2024-11-10 09:16:01,518 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 2024-11-10 09:16:01,537 main DEBUG Took 0.015892 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 09:16:01,537 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 09:16:01,548 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 09:16:01,550 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 09:16:01,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,566 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 09:16:01,590 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,594 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,595 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,611 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,619 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,625 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,627 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,630 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,634 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,635 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,638 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:16:01,639 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,640 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 09:16:01,642 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:16:01,644 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 09:16:01,648 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 09:16:01,648 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 09:16:01,650 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 09:16:01,651 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 09:16:01,666 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 09:16:01,670 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 09:16:01,673 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 09:16:01,673 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 09:16:01,674 main DEBUG createAppenders(={Console}) 2024-11-10 09:16:01,675 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 initialized 2024-11-10 09:16:01,675 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 2024-11-10 09:16:01,676 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 OK. 2024-11-10 09:16:01,677 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 09:16:01,677 main DEBUG OutputStream closed 2024-11-10 09:16:01,678 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 09:16:01,678 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 09:16:01,678 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4c1909a3 OK 2024-11-10 09:16:01,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 09:16:01,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 09:16:01,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 09:16:01,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 09:16:01,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 09:16:01,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 09:16:01,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 09:16:01,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 09:16:01,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 09:16:01,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 09:16:01,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 09:16:01,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 09:16:01,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 09:16:01,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 09:16:01,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 09:16:01,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 09:16:01,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 09:16:01,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 09:16:01,856 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 09:16:01,857 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@bcef303) with optional ClassLoader: null 2024-11-10 09:16:01,857 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 09:16:01,859 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@bcef303] started OK. 2024-11-10T09:16:02,235 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd 2024-11-10 09:16:02,239 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 09:16:02,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T09:16:02,252 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-10T09:16:02,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T09:16:02,331 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e, deleteOnExit=true 2024-11-10T09:16:02,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T09:16:02,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/test.cache.data in system properties and HBase conf 2024-11-10T09:16:02,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T09:16:02,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/hadoop.log.dir in system properties and HBase conf 2024-11-10T09:16:02,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T09:16:02,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T09:16:02,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T09:16:02,469 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T09:16:02,599 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T09:16:02,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T09:16:02,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T09:16:02,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T09:16:02,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T09:16:02,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T09:16:02,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T09:16:02,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T09:16:02,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T09:16:02,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T09:16:02,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/nfs.dump.dir in system properties and HBase conf 2024-11-10T09:16:02,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/java.io.tmpdir in system properties and HBase conf 2024-11-10T09:16:02,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T09:16:02,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T09:16:02,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T09:16:03,832 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T09:16:03,922 INFO [Time-limited test {}] log.Log(170): Logging initialized @3515ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T09:16:04,021 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T09:16:04,079 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T09:16:04,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T09:16:04,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T09:16:04,100 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T09:16:04,111 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T09:16:04,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/hadoop.log.dir/,AVAILABLE} 2024-11-10T09:16:04,115 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T09:16:04,281 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/java.io.tmpdir/jetty-localhost-37671-hadoop-hdfs-3_4_1-tests_jar-_-any-4832661506548791524/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T09:16:04,296 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:37671} 2024-11-10T09:16:04,297 INFO [Time-limited test {}] server.Server(415): Started @3891ms 2024-11-10T09:16:04,931 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T09:16:04,942 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T09:16:04,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T09:16:04,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T09:16:04,947 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T09:16:04,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/hadoop.log.dir/,AVAILABLE} 2024-11-10T09:16:04,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T09:16:05,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/java.io.tmpdir/jetty-localhost-34153-hadoop-hdfs-3_4_1-tests_jar-_-any-6938907143182870633/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T09:16:05,085 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:34153} 2024-11-10T09:16:05,085 INFO [Time-limited test {}] server.Server(415): Started @4679ms 2024-11-10T09:16:05,155 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T09:16:06,596 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/data/data2/current/BP-1750528295-172.17.0.3-1731230163282/current, will proceed with Du for space computation calculation, 2024-11-10T09:16:06,596 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/data/data1/current/BP-1750528295-172.17.0.3-1731230163282/current, will proceed with Du for space computation calculation, 2024-11-10T09:16:06,657 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T09:16:06,722 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9cd9cbee7f9fa3f3 with lease ID 0x2a3aa76a02f4afe7: Processing first storage report for DS-60d05f61-4b7d-451a-9272-881ec8cad4c0 from datanode DatanodeRegistration(127.0.0.1:39443, datanodeUuid=93b0e4a0-6ef8-482f-96c7-e988491df6b4, infoPort=44341, infoSecurePort=0, ipcPort=45281, storageInfo=lv=-57;cid=testClusterID;nsid=393488696;c=1731230163282) 2024-11-10T09:16:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9cd9cbee7f9fa3f3 with lease ID 0x2a3aa76a02f4afe7: from storage DS-60d05f61-4b7d-451a-9272-881ec8cad4c0 node DatanodeRegistration(127.0.0.1:39443, datanodeUuid=93b0e4a0-6ef8-482f-96c7-e988491df6b4, infoPort=44341, infoSecurePort=0, ipcPort=45281, storageInfo=lv=-57;cid=testClusterID;nsid=393488696;c=1731230163282), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T09:16:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9cd9cbee7f9fa3f3 with lease ID 0x2a3aa76a02f4afe7: Processing first storage report for DS-64883513-25f8-4584-9a53-a2906818bd42 from datanode DatanodeRegistration(127.0.0.1:39443, datanodeUuid=93b0e4a0-6ef8-482f-96c7-e988491df6b4, infoPort=44341, infoSecurePort=0, ipcPort=45281, storageInfo=lv=-57;cid=testClusterID;nsid=393488696;c=1731230163282) 2024-11-10T09:16:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9cd9cbee7f9fa3f3 with lease ID 0x2a3aa76a02f4afe7: from storage DS-64883513-25f8-4584-9a53-a2906818bd42 node DatanodeRegistration(127.0.0.1:39443, datanodeUuid=93b0e4a0-6ef8-482f-96c7-e988491df6b4, infoPort=44341, infoSecurePort=0, ipcPort=45281, storageInfo=lv=-57;cid=testClusterID;nsid=393488696;c=1731230163282), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T09:16:06,808 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd 2024-11-10T09:16:06,923 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/zookeeper_0, clientPort=53423, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T09:16:06,934 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53423 2024-11-10T09:16:06,948 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:06,951 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741825_1001 (size=7) 2024-11-10T09:16:07,577 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 with version=8 2024-11-10T09:16:07,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/hbase-staging 2024-11-10T09:16:07,667 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T09:16:07,917 INFO [Time-limited test {}] client.ConnectionUtils(128): master/802346a334f5:0 server-side Connection retries=45 2024-11-10T09:16:07,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:16:07,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T09:16:07,932 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T09:16:07,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:16:07,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T09:16:08,095 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T09:16:08,159 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T09:16:08,169 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T09:16:08,173 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T09:16:08,200 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8165 (auto-detected) 2024-11-10T09:16:08,201 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-10T09:16:08,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43719 2024-11-10T09:16:08,255 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43719 connecting to ZooKeeper ensemble=127.0.0.1:53423 2024-11-10T09:16:08,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:437190x0, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T09:16:08,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43719-0x1012407eb210000 connected 2024-11-10T09:16:08,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:08,430 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:08,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:16:08,444 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11, hbase.cluster.distributed=false 2024-11-10T09:16:08,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T09:16:08,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43719 2024-11-10T09:16:08,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43719 2024-11-10T09:16:08,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43719 2024-11-10T09:16:08,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43719 2024-11-10T09:16:08,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43719 2024-11-10T09:16:08,568 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/802346a334f5:0 server-side Connection retries=45 2024-11-10T09:16:08,569 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:16:08,569 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T09:16:08,570 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T09:16:08,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:16:08,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T09:16:08,572 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T09:16:08,575 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T09:16:08,575 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38513 2024-11-10T09:16:08,577 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38513 connecting to ZooKeeper ensemble=127.0.0.1:53423 2024-11-10T09:16:08,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:08,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:08,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385130x0, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T09:16:08,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385130x0, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:16:08,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38513-0x1012407eb210001 connected 2024-11-10T09:16:08,598 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T09:16:08,605 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T09:16:08,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T09:16:08,612 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T09:16:08,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38513 2024-11-10T09:16:08,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38513 2024-11-10T09:16:08,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38513 2024-11-10T09:16:08,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38513 2024-11-10T09:16:08,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38513 2024-11-10T09:16:08,630 DEBUG [M:0;802346a334f5:43719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;802346a334f5:43719 2024-11-10T09:16:08,631 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/802346a334f5,43719,1731230167759 2024-11-10T09:16:08,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:16:08,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:16:08,650 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/802346a334f5,43719,1731230167759 2024-11-10T09:16:08,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:08,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T09:16:08,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:08,680 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T09:16:08,681 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/802346a334f5,43719,1731230167759 from backup master directory 2024-11-10T09:16:08,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:16:08,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/802346a334f5,43719,1731230167759 2024-11-10T09:16:08,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:16:08,690 WARN [master/802346a334f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T09:16:08,690 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=802346a334f5,43719,1731230167759 2024-11-10T09:16:08,692 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T09:16:08,693 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T09:16:08,745 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/hbase.id] with ID: 7a1f5e9f-ae4c-4e58-873b-13922a881c13 2024-11-10T09:16:08,745 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/.tmp/hbase.id 2024-11-10T09:16:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741826_1002 (size=42) 2024-11-10T09:16:09,159 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/.tmp/hbase.id]:[hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/hbase.id] 2024-11-10T09:16:09,208 INFO [master/802346a334f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:16:09,214 INFO [master/802346a334f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T09:16:09,236 INFO [master/802346a334f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-10T09:16:09,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:09,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741827_1003 (size=196) 2024-11-10T09:16:09,744 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:16:09,747 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T09:16:09,762 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:09,766 INFO [master/802346a334f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T09:16:09,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741828_1004 (size=1189) 2024-11-10T09:16:10,211 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store 2024-11-10T09:16:10,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741829_1005 (size=34) 2024-11-10T09:16:10,641 INFO [master/802346a334f5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T09:16:10,645 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:10,647 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T09:16:10,647 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:16:10,647 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:16:10,649 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T09:16:10,649 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:16:10,649 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:16:10,651 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731230170646Disabling compacts and flushes for region at 1731230170646Disabling writes for close at 1731230170649 (+3 ms)Writing region close event to WAL at 1731230170649Closed at 1731230170649 2024-11-10T09:16:10,653 WARN [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/.initializing 2024-11-10T09:16:10,653 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/WALs/802346a334f5,43719,1731230167759 2024-11-10T09:16:10,661 INFO [master/802346a334f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T09:16:10,675 INFO [master/802346a334f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=802346a334f5%2C43719%2C1731230167759, suffix=, logDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/WALs/802346a334f5,43719,1731230167759, archiveDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/oldWALs, maxLogs=10 2024-11-10T09:16:10,697 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/WALs/802346a334f5,43719,1731230167759/802346a334f5%2C43719%2C1731230167759.1731230170679, exclude list is [], retry=0 2024-11-10T09:16:10,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39443,DS-60d05f61-4b7d-451a-9272-881ec8cad4c0,DISK] 2024-11-10T09:16:10,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T09:16:10,751 INFO [master/802346a334f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/WALs/802346a334f5,43719,1731230167759/802346a334f5%2C43719%2C1731230167759.1731230170679 2024-11-10T09:16:10,752 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44341:44341)] 2024-11-10T09:16:10,752 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:16:10,753 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:10,756 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,756 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T09:16:10,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:10,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:10,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T09:16:10,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:10,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:10,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T09:16:10,833 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:10,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:10,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,838 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T09:16:10,838 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:10,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:10,840 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,844 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,845 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,850 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,851 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,855 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T09:16:10,860 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:16:10,866 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:16:10,867 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62888052, jitterRate=-0.06289499998092651}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T09:16:10,874 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731230170769Initializing all the Stores at 1731230170771 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230170772 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230170772Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230170773 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230170773Cleaning up temporary data from old regions at 1731230170851 (+78 ms)Region opened successfully at 1731230170873 (+22 ms) 2024-11-10T09:16:10,875 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T09:16:10,915 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29efa2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=802346a334f5/172.17.0.3:0 2024-11-10T09:16:10,956 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T09:16:10,971 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T09:16:10,971 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T09:16:10,974 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T09:16:10,975 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T09:16:10,980 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T09:16:10,980 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T09:16:11,011 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T09:16:11,021 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T09:16:11,033 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T09:16:11,035 INFO [master/802346a334f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T09:16:11,037 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T09:16:11,047 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T09:16:11,049 INFO [master/802346a334f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T09:16:11,052 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T09:16:11,064 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T09:16:11,066 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T09:16:11,078 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T09:16:11,095 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T09:16:11,106 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T09:16:11,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T09:16:11,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T09:16:11,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:11,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:11,123 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=802346a334f5,43719,1731230167759, sessionid=0x1012407eb210000, setting cluster-up flag (Was=false) 2024-11-10T09:16:11,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:11,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:11,183 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T09:16:11,186 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=802346a334f5,43719,1731230167759 2024-11-10T09:16:11,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:11,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:11,246 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T09:16:11,249 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=802346a334f5,43719,1731230167759 2024-11-10T09:16:11,256 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T09:16:11,321 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(746): ClusterId : 7a1f5e9f-ae4c-4e58-873b-13922a881c13 2024-11-10T09:16:11,324 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T09:16:11,328 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T09:16:11,340 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T09:16:11,340 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T09:16:11,342 INFO [master/802346a334f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T09:16:11,348 INFO [master/802346a334f5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T09:16:11,353 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T09:16:11,354 DEBUG [RS:0;802346a334f5:38513 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513c3e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=802346a334f5/172.17.0.3:0 2024-11-10T09:16:11,354 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 802346a334f5,43719,1731230167759 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T09:16:11,362 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:16:11,362 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:16:11,362 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:16:11,362 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:16:11,362 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/802346a334f5:0, corePoolSize=10, maxPoolSize=10 2024-11-10T09:16:11,362 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,363 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/802346a334f5:0, corePoolSize=2, maxPoolSize=2 2024-11-10T09:16:11,363 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,364 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731230201364 2024-11-10T09:16:11,366 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T09:16:11,367 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T09:16:11,368 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T09:16:11,368 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T09:16:11,370 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T09:16:11,371 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T09:16:11,371 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T09:16:11,372 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T09:16:11,372 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,375 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:11,375 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;802346a334f5:38513 2024-11-10T09:16:11,375 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T09:16:11,376 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T09:16:11,378 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T09:16:11,378 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T09:16:11,379 INFO [RS:0;802346a334f5:38513 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T09:16:11,379 INFO [RS:0;802346a334f5:38513 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T09:16:11,380 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T09:16:11,380 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T09:16:11,381 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T09:16:11,383 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(2659): reportForDuty to master=802346a334f5,43719,1731230167759 with port=38513, startcode=1731230168538 2024-11-10T09:16:11,385 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.large.0-1731230171382,5,FailOnTimeoutGroup] 2024-11-10T09:16:11,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741831_1007 (size=1321) 2024-11-10T09:16:11,389 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.small.0-1731230171385,5,FailOnTimeoutGroup] 2024-11-10T09:16:11,390 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,390 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T09:16:11,391 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,391 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,398 DEBUG [RS:0;802346a334f5:38513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T09:16:11,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43873, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T09:16:11,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 802346a334f5,38513,1731230168538 2024-11-10T09:16:11,438 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43719 {}] master.ServerManager(517): Registering regionserver=802346a334f5,38513,1731230168538 2024-11-10T09:16:11,452 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:16:11,452 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38659 2024-11-10T09:16:11,452 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T09:16:11,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T09:16:11,468 DEBUG [RS:0;802346a334f5:38513 {}] zookeeper.ZKUtil(111): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/802346a334f5,38513,1731230168538 2024-11-10T09:16:11,469 WARN [RS:0;802346a334f5:38513 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T09:16:11,469 INFO [RS:0;802346a334f5:38513 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T09:16:11,469 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538 2024-11-10T09:16:11,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [802346a334f5,38513,1731230168538] 2024-11-10T09:16:11,493 INFO [RS:0;802346a334f5:38513 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T09:16:11,509 INFO [RS:0;802346a334f5:38513 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T09:16:11,513 INFO [RS:0;802346a334f5:38513 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T09:16:11,513 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,514 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T09:16:11,519 INFO [RS:0;802346a334f5:38513 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T09:16:11,520 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,521 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,521 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,521 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,521 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,521 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,521 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/802346a334f5:0, corePoolSize=2, maxPoolSize=2 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:16:11,522 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/802346a334f5:0, corePoolSize=3, maxPoolSize=3 2024-11-10T09:16:11,523 DEBUG [RS:0;802346a334f5:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0, corePoolSize=3, maxPoolSize=3 2024-11-10T09:16:11,523 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,524 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,524 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,524 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,524 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,524 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38513,1731230168538-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T09:16:11,542 INFO [RS:0;802346a334f5:38513 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T09:16:11,544 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38513,1731230168538-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,544 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,544 INFO [RS:0;802346a334f5:38513 {}] regionserver.Replication(171): 802346a334f5,38513,1731230168538 started 2024-11-10T09:16:11,564 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:11,565 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1482): Serving as 802346a334f5,38513,1731230168538, RpcServer on 802346a334f5/172.17.0.3:38513, sessionid=0x1012407eb210001 2024-11-10T09:16:11,565 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T09:16:11,566 DEBUG [RS:0;802346a334f5:38513 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 802346a334f5,38513,1731230168538 2024-11-10T09:16:11,566 DEBUG [RS:0;802346a334f5:38513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '802346a334f5,38513,1731230168538' 2024-11-10T09:16:11,566 DEBUG [RS:0;802346a334f5:38513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T09:16:11,567 DEBUG [RS:0;802346a334f5:38513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T09:16:11,568 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T09:16:11,568 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T09:16:11,568 DEBUG [RS:0;802346a334f5:38513 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 802346a334f5,38513,1731230168538 2024-11-10T09:16:11,568 DEBUG [RS:0;802346a334f5:38513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '802346a334f5,38513,1731230168538' 2024-11-10T09:16:11,568 DEBUG [RS:0;802346a334f5:38513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T09:16:11,569 DEBUG [RS:0;802346a334f5:38513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T09:16:11,570 DEBUG [RS:0;802346a334f5:38513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T09:16:11,570 INFO [RS:0;802346a334f5:38513 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T09:16:11,570 INFO [RS:0;802346a334f5:38513 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T09:16:11,676 INFO [RS:0;802346a334f5:38513 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T09:16:11,680 INFO [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=802346a334f5%2C38513%2C1731230168538, suffix=, logDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538, archiveDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/oldWALs, maxLogs=32 2024-11-10T09:16:11,696 DEBUG [RS:0;802346a334f5:38513 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538/802346a334f5%2C38513%2C1731230168538.1731230171683, exclude list is [], retry=0 2024-11-10T09:16:11,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39443,DS-60d05f61-4b7d-451a-9272-881ec8cad4c0,DISK] 2024-11-10T09:16:11,703 INFO [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538/802346a334f5%2C38513%2C1731230168538.1731230171683 2024-11-10T09:16:11,704 DEBUG [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44341:44341)] 2024-11-10T09:16:11,791 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T09:16:11,791 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:16:11,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741833_1009 (size=32) 2024-11-10T09:16:12,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:12,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T09:16:12,215 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T09:16:12,216 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T09:16:12,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T09:16:12,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T09:16:12,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T09:16:12,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T09:16:12,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T09:16:12,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T09:16:12,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740 2024-11-10T09:16:12,234 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740 2024-11-10T09:16:12,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T09:16:12,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T09:16:12,239 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:16:12,244 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T09:16:12,249 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:16:12,250 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59565344, jitterRate=-0.11240720748901367}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:16:12,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731230172207Initializing all the Stores at 1731230172210 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230172210Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230172210Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230172211 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230172211Cleaning up temporary data from old regions at 1731230172238 (+27 ms)Region opened successfully at 1731230172255 (+17 ms) 2024-11-10T09:16:12,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T09:16:12,255 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T09:16:12,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T09:16:12,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T09:16:12,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T09:16:12,257 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T09:16:12,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731230172255Disabling compacts and flushes for region at 1731230172255Disabling writes for close at 1731230172255Writing region close event to WAL at 1731230172256 (+1 ms)Closed at 1731230172257 (+1 ms) 2024-11-10T09:16:12,260 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T09:16:12,260 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T09:16:12,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T09:16:12,274 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T09:16:12,278 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T09:16:12,431 DEBUG [802346a334f5:43719 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T09:16:12,442 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:12,450 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 802346a334f5,38513,1731230168538, state=OPENING 2024-11-10T09:16:12,544 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T09:16:12,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:12,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:16:12,570 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:16:12,570 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:16:12,572 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T09:16:12,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:16:12,750 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T09:16:12,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33777, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T09:16:12,763 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T09:16:12,764 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T09:16:12,764 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T09:16:12,768 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=802346a334f5%2C38513%2C1731230168538.meta, suffix=.meta, logDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538, archiveDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/oldWALs, maxLogs=32 2024-11-10T09:16:12,782 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538/802346a334f5%2C38513%2C1731230168538.meta.1731230172769.meta, exclude list is [], retry=0 2024-11-10T09:16:12,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39443,DS-60d05f61-4b7d-451a-9272-881ec8cad4c0,DISK] 2024-11-10T09:16:12,789 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/WALs/802346a334f5,38513,1731230168538/802346a334f5%2C38513%2C1731230168538.meta.1731230172769.meta 2024-11-10T09:16:12,789 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44341:44341)] 2024-11-10T09:16:12,789 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:16:12,791 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T09:16:12,793 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T09:16:12,798 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T09:16:12,802 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T09:16:12,802 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:12,803 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T09:16:12,803 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T09:16:12,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T09:16:12,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T09:16:12,807 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T09:16:12,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T09:16:12,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T09:16:12,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T09:16:12,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T09:16:12,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T09:16:12,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:12,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:16:12,816 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T09:16:12,818 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740 2024-11-10T09:16:12,821 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740 2024-11-10T09:16:12,823 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T09:16:12,823 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T09:16:12,824 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:16:12,827 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T09:16:12,828 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64149630, jitterRate=-0.044096022844314575}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:16:12,829 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T09:16:12,830 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731230172803Writing region info on filesystem at 1731230172803Initializing all the Stores at 1731230172805 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230172805Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230172806 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230172806Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230172806Cleaning up temporary data from old regions at 1731230172823 (+17 ms)Running coprocessor post-open hooks at 1731230172829 (+6 ms)Region opened successfully at 1731230172830 (+1 ms) 2024-11-10T09:16:12,839 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731230172742 2024-11-10T09:16:12,855 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T09:16:12,856 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T09:16:12,858 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:12,861 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 802346a334f5,38513,1731230168538, state=OPEN 2024-11-10T09:16:12,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T09:16:12,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T09:16:12,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:16:12,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:16:12,960 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:12,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T09:16:12,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=802346a334f5,38513,1731230168538 in 387 msec 2024-11-10T09:16:12,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T09:16:12,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 706 msec 2024-11-10T09:16:12,980 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T09:16:12,980 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T09:16:13,006 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:13,008 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:13,037 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:13,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:13,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7780 sec 2024-11-10T09:16:13,066 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731230173066, completionTime=-1 2024-11-10T09:16:13,070 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T09:16:13,070 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T09:16:13,105 INFO [master/802346a334f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T09:16:13,105 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731230233105 2024-11-10T09:16:13,106 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731230293106 2024-11-10T09:16:13,106 INFO [master/802346a334f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 35 msec 2024-11-10T09:16:13,109 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,43719,1731230167759-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,110 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,43719,1731230167759-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,110 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,43719,1731230167759-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,112 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-802346a334f5:43719, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,112 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,113 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,118 DEBUG [master/802346a334f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T09:16:13,144 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.454sec 2024-11-10T09:16:13,146 INFO [master/802346a334f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T09:16:13,147 INFO [master/802346a334f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T09:16:13,148 INFO [master/802346a334f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T09:16:13,149 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T09:16:13,149 INFO [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T09:16:13,150 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,43719,1731230167759-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T09:16:13,150 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,43719,1731230167759-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T09:16:13,159 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T09:16:13,160 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T09:16:13,160 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,43719,1731230167759-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:16:13,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:13,238 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T09:16:13,238 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T09:16:13,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:13,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:13,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:13,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:13,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:13,273 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:13,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:13,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=802346a334f5,43719,1731230167759 2024-11-10T09:16:13,325 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=663, ProcessCount=11, AvailableMemoryMB=3516 2024-11-10T09:16:13,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:16:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:16:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-10T09:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:16:13,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:16:13,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:16:13,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:13,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:16:13,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T09:16:13,402 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:13,407 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:16:13,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741835_1011 (size=1061) 2024-11-10T09:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:16:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:16:13,843 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:16:13,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741836_1012 (size=53) 2024-11-10T09:16:13,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:13,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bfb4913e15c1cd99308737d49021ea5d, disabling compactions & flushes 2024-11-10T09:16:13,858 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:13,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:13,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. after waiting 0 ms 2024-11-10T09:16:13,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:13,858 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:13,859 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bfb4913e15c1cd99308737d49021ea5d: Waiting for close lock at 1731230173858Disabling compacts and flushes for region at 1731230173858Disabling writes for close at 1731230173858Writing region close event to WAL at 1731230173858Closed at 1731230173858 2024-11-10T09:16:13,861 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:16:13,867 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230173861"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230173861"}]},"ts":"1731230173861"} 2024-11-10T09:16:13,871 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:16:13,873 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:16:13,875 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230173873"}]},"ts":"1731230173873"} 2024-11-10T09:16:13,880 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:16:13,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, ASSIGN}] 2024-11-10T09:16:13,884 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, ASSIGN 2024-11-10T09:16:13,886 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, ASSIGN; state=OFFLINE, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=false 2024-11-10T09:16:14,039 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bfb4913e15c1cd99308737d49021ea5d, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:14,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, ASSIGN because future has completed 2024-11-10T09:16:14,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:16:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:16:14,210 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:14,210 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:16:14,210 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,210 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:14,211 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,211 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,213 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,216 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:14,217 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfb4913e15c1cd99308737d49021ea5d columnFamilyName A 2024-11-10T09:16:14,217 DEBUG [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:14,218 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.HStore(327): Store=bfb4913e15c1cd99308737d49021ea5d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:14,218 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,220 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:14,221 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfb4913e15c1cd99308737d49021ea5d columnFamilyName B 2024-11-10T09:16:14,221 DEBUG [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:14,222 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.HStore(327): Store=bfb4913e15c1cd99308737d49021ea5d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:14,222 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,225 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:14,225 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfb4913e15c1cd99308737d49021ea5d columnFamilyName C 2024-11-10T09:16:14,225 DEBUG [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:14,226 INFO [StoreOpener-bfb4913e15c1cd99308737d49021ea5d-1 {}] regionserver.HStore(327): Store=bfb4913e15c1cd99308737d49021ea5d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:14,227 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:14,227 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,229 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,230 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,232 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,232 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,233 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:16:14,236 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,240 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:16:14,241 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bfb4913e15c1cd99308737d49021ea5d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74980372, jitterRate=0.11729460954666138}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:16:14,241 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,243 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bfb4913e15c1cd99308737d49021ea5d: Running coprocessor pre-open hook at 1731230174211Writing region info on filesystem at 1731230174211Initializing all the Stores at 1731230174212 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230174212Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230174212Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230174212Cleaning up temporary data from old regions at 1731230174232 (+20 ms)Running coprocessor post-open hooks at 1731230174241 (+9 ms)Region opened successfully at 1731230174243 (+2 ms) 2024-11-10T09:16:14,245 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., pid=6, masterSystemTime=1731230174204 2024-11-10T09:16:14,249 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:14,249 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:14,250 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bfb4913e15c1cd99308737d49021ea5d, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:14,252 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43719 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=802346a334f5,38513,1731230168538, table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T09:16:14,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:16:14,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T09:16:14,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 in 207 msec 2024-11-10T09:16:14,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T09:16:14,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, ASSIGN in 379 msec 2024-11-10T09:16:14,268 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:16:14,268 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230174268"}]},"ts":"1731230174268"} 2024-11-10T09:16:14,272 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:16:14,274 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:16:14,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-10T09:16:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:16:14,570 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:14,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,586 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,593 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:16:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:16:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-11-10T09:16:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:16:14,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:16:14,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,606 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,610 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,613 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,617 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,620 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,625 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,629 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,644 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5346dc42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:14,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:14,667 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:14,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:14,677 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,677 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,683 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T09:16:14,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,696 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:14,699 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,702 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:16:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,713 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,715 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,715 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,721 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,725 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,729 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:14,729 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,730 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:14,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:16:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:16:14,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:16:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:14,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:14,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:14,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:14,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:14,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:14,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:16:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:14,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:14,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:14,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/60a79dfd262e4998bdabd23eb0ad30d4 is 50, key is test_row_0/A:col10/1731230174790/Put/seqid=0 2024-11-10T09:16:15,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741837_1013 (size=14341) 2024-11-10T09:16:15,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/60a79dfd262e4998bdabd23eb0ad30d4 2024-11-10T09:16:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:16:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230234980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230234980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230235012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230235016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230235034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:16:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:15,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/aca3267b70104dc7aa98b4f62bf26c63 is 50, key is test_row_0/B:col10/1731230174790/Put/seqid=0 2024-11-10T09:16:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741838_1014 (size=12001) 2024-11-10T09:16:15,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/aca3267b70104dc7aa98b4f62bf26c63 2024-11-10T09:16:15,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230235238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230235239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230235240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230235240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230235244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:16:15,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/15c25d28d20c4d9bad127e13d4680302 is 50, key is test_row_0/C:col10/1731230174790/Put/seqid=0 2024-11-10T09:16:15,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:15,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:15,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741839_1015 (size=12001) 2024-11-10T09:16:15,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/15c25d28d20c4d9bad127e13d4680302 2024-11-10T09:16:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/60a79dfd262e4998bdabd23eb0ad30d4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/60a79dfd262e4998bdabd23eb0ad30d4 2024-11-10T09:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:16:15,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/60a79dfd262e4998bdabd23eb0ad30d4, entries=200, sequenceid=13, filesize=14.0 K 2024-11-10T09:16:15,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/aca3267b70104dc7aa98b4f62bf26c63 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca3267b70104dc7aa98b4f62bf26c63 2024-11-10T09:16:15,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca3267b70104dc7aa98b4f62bf26c63, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:16:15,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/15c25d28d20c4d9bad127e13d4680302 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/15c25d28d20c4d9bad127e13d4680302 2024-11-10T09:16:15,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/15c25d28d20c4d9bad127e13d4680302, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:16:15,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bfb4913e15c1cd99308737d49021ea5d in 598ms, sequenceid=13, compaction requested=false 2024-11-10T09:16:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T09:16:15,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:15,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:16:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:15,438 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:16:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:15,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:15,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:15,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:15,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:15,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:15,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/dd7166a1764e41b89cbe277a85633925 is 50, key is test_row_0/A:col10/1731230175014/Put/seqid=0 2024-11-10T09:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:15,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:15,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741840_1016 (size=12001) 2024-11-10T09:16:15,483 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/dd7166a1764e41b89cbe277a85633925 2024-11-10T09:16:15,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/98566a4f78d349729f9b16b706e3fb68 is 50, key is test_row_0/B:col10/1731230175014/Put/seqid=0 2024-11-10T09:16:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741841_1017 (size=12001) 2024-11-10T09:16:15,523 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/98566a4f78d349729f9b16b706e3fb68 2024-11-10T09:16:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230235494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230235500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230235517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230235535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230235538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/e2a9563484d14610a68670fabeeedaa7 is 50, key is test_row_0/C:col10/1731230175014/Put/seqid=0 2024-11-10T09:16:15,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741842_1018 (size=12001) 2024-11-10T09:16:15,606 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/e2a9563484d14610a68670fabeeedaa7 2024-11-10T09:16:15,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/dd7166a1764e41b89cbe277a85633925 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dd7166a1764e41b89cbe277a85633925 2024-11-10T09:16:15,645 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dd7166a1764e41b89cbe277a85633925, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T09:16:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230235638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/98566a4f78d349729f9b16b706e3fb68 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/98566a4f78d349729f9b16b706e3fb68 2024-11-10T09:16:15,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230235649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230235651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,665 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/98566a4f78d349729f9b16b706e3fb68, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T09:16:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230235658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230235659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/e2a9563484d14610a68670fabeeedaa7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e2a9563484d14610a68670fabeeedaa7 2024-11-10T09:16:15,687 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e2a9563484d14610a68670fabeeedaa7, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T09:16:15,695 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for bfb4913e15c1cd99308737d49021ea5d in 257ms, sequenceid=33, compaction requested=false 2024-11-10T09:16:15,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:15,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:15,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T09:16:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T09:16:15,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T09:16:15,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-10T09:16:15,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0290 sec 2024-11-10T09:16:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:16:15,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:15,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T09:16:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:16:15,854 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:15,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:15,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:15,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:16:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:15,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:15,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/2b01e31db0254331ad3c071b500ad4a2 is 50, key is test_row_0/A:col10/1731230175863/Put/seqid=0 2024-11-10T09:16:15,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741843_1019 (size=14341) 2024-11-10T09:16:15,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/2b01e31db0254331ad3c071b500ad4a2 2024-11-10T09:16:15,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230235907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230235910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230235914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230235920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230235922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:15,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:15,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/e7c15d2f19f74f87a48a0fe830966145 is 50, key is test_row_0/B:col10/1731230175863/Put/seqid=0 2024-11-10T09:16:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:16:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741844_1020 (size=12001) 2024-11-10T09:16:16,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230236028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230236029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230236038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230236039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230236039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:16:16,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230236239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230236240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230236258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230236259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230236259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/e7c15d2f19f74f87a48a0fe830966145 2024-11-10T09:16:16,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/8d47f4b5920f43c394251bc434b48007 is 50, key is test_row_0/C:col10/1731230175863/Put/seqid=0 2024-11-10T09:16:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741845_1021 (size=12001) 2024-11-10T09:16:16,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/8d47f4b5920f43c394251bc434b48007 2024-11-10T09:16:16,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/2b01e31db0254331ad3c071b500ad4a2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/2b01e31db0254331ad3c071b500ad4a2 2024-11-10T09:16:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:16:16,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/2b01e31db0254331ad3c071b500ad4a2, entries=200, sequenceid=48, filesize=14.0 K 2024-11-10T09:16:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/e7c15d2f19f74f87a48a0fe830966145 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/e7c15d2f19f74f87a48a0fe830966145 2024-11-10T09:16:16,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/e7c15d2f19f74f87a48a0fe830966145, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T09:16:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/8d47f4b5920f43c394251bc434b48007 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/8d47f4b5920f43c394251bc434b48007 2024-11-10T09:16:16,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/8d47f4b5920f43c394251bc434b48007, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T09:16:16,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for bfb4913e15c1cd99308737d49021ea5d in 674ms, sequenceid=48, compaction requested=true 2024-11-10T09:16:16,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:16,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:16,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:16,547 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:16,548 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:16,560 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:16,560 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:16,562 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:16,562 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,563 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:16,563 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,563 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/60a79dfd262e4998bdabd23eb0ad30d4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dd7166a1764e41b89cbe277a85633925, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/2b01e31db0254331ad3c071b500ad4a2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=39.7 K 2024-11-10T09:16:16,563 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca3267b70104dc7aa98b4f62bf26c63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/98566a4f78d349729f9b16b706e3fb68, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/e7c15d2f19f74f87a48a0fe830966145] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=35.2 K 2024-11-10T09:16:16,565 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting aca3267b70104dc7aa98b4f62bf26c63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230174750 2024-11-10T09:16:16,566 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60a79dfd262e4998bdabd23eb0ad30d4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230174737 2024-11-10T09:16:16,566 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd7166a1764e41b89cbe277a85633925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731230174976 2024-11-10T09:16:16,567 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 98566a4f78d349729f9b16b706e3fb68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731230174976 2024-11-10T09:16:16,567 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b01e31db0254331ad3c071b500ad4a2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731230175494 2024-11-10T09:16:16,570 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c15d2f19f74f87a48a0fe830966145, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731230175510 2024-11-10T09:16:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:16,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:16:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:16,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/bfb554bb12514c88a255cd1b1933b396 is 50, key is test_row_0/A:col10/1731230175915/Put/seqid=0 2024-11-10T09:16:16,634 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#11 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:16,634 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#10 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:16,635 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/596ccf60ac954237a4c1b422a77c6824 is 50, key is test_row_0/A:col10/1731230175863/Put/seqid=0 2024-11-10T09:16:16,635 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/0d591b6007654381a9bc1151d3c44517 is 50, key is test_row_0/B:col10/1731230175863/Put/seqid=0 2024-11-10T09:16:16,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741846_1022 (size=16681) 2024-11-10T09:16:16,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741847_1023 (size=12104) 2024-11-10T09:16:16,687 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/0d591b6007654381a9bc1151d3c44517 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0d591b6007654381a9bc1151d3c44517 2024-11-10T09:16:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230236661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230236665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741848_1024 (size=12104) 2024-11-10T09:16:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230236669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230236673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230236675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,719 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/596ccf60ac954237a4c1b422a77c6824 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/596ccf60ac954237a4c1b422a77c6824 2024-11-10T09:16:16,724 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 0d591b6007654381a9bc1151d3c44517(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:16,724 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:16,727 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230176545; duration=0sec 2024-11-10T09:16:16,728 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:16,728 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:16,728 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:16,736 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:16,736 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:16,736 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 596ccf60ac954237a4c1b422a77c6824(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:16,736 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,736 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:16,736 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230176539; duration=0sec 2024-11-10T09:16:16,736 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/15c25d28d20c4d9bad127e13d4680302, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e2a9563484d14610a68670fabeeedaa7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/8d47f4b5920f43c394251bc434b48007] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=35.2 K 2024-11-10T09:16:16,736 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:16,736 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:16,739 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c25d28d20c4d9bad127e13d4680302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230174750 2024-11-10T09:16:16,741 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e2a9563484d14610a68670fabeeedaa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731230174976 2024-11-10T09:16:16,743 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d47f4b5920f43c394251bc434b48007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731230175510 2024-11-10T09:16:16,791 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#12 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:16,791 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/585639ad15ac443f8eb7b2bd34c003d1 is 50, key is test_row_0/C:col10/1731230175863/Put/seqid=0 2024-11-10T09:16:16,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230236798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230236799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230236799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230236808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230236799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741849_1025 (size=12104) 2024-11-10T09:16:16,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:16,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:16:17,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230237020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230237028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230237028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230237029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230237029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/bfb554bb12514c88a255cd1b1933b396 2024-11-10T09:16:17,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/bb5cda50b9c54a739c53a055d575b691 is 50, key is test_row_0/B:col10/1731230175915/Put/seqid=0 2024-11-10T09:16:17,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741850_1026 (size=12001) 2024-11-10T09:16:17,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/bb5cda50b9c54a739c53a055d575b691 2024-11-10T09:16:17,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:17,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:17,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:17,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/4bc38bde595947c2bdcf5c65b459483e is 50, key is test_row_0/C:col10/1731230175915/Put/seqid=0 2024-11-10T09:16:17,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741851_1027 (size=12001) 2024-11-10T09:16:17,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/4bc38bde595947c2bdcf5c65b459483e 2024-11-10T09:16:17,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/bfb554bb12514c88a255cd1b1933b396 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bfb554bb12514c88a255cd1b1933b396 2024-11-10T09:16:17,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bfb554bb12514c88a255cd1b1933b396, entries=250, sequenceid=67, filesize=16.3 K 2024-11-10T09:16:17,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/bb5cda50b9c54a739c53a055d575b691 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/bb5cda50b9c54a739c53a055d575b691 2024-11-10T09:16:17,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/bb5cda50b9c54a739c53a055d575b691, entries=150, sequenceid=67, filesize=11.7 K 2024-11-10T09:16:17,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/4bc38bde595947c2bdcf5c65b459483e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/4bc38bde595947c2bdcf5c65b459483e 2024-11-10T09:16:17,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/4bc38bde595947c2bdcf5c65b459483e, entries=150, sequenceid=67, filesize=11.7 K 2024-11-10T09:16:17,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for bfb4913e15c1cd99308737d49021ea5d in 639ms, sequenceid=67, compaction requested=false 2024-11-10T09:16:17,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:17,261 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/585639ad15ac443f8eb7b2bd34c003d1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/585639ad15ac443f8eb7b2bd34c003d1 2024-11-10T09:16:17,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:16:17,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:17,276 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:16:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:17,281 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into 585639ad15ac443f8eb7b2bd34c003d1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:17,282 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:17,282 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230176556; duration=0sec 2024-11-10T09:16:17,282 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:17,282 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/5d44ed13b7b5487a8f8b2958b9bf0ba3 is 50, key is test_row_0/A:col10/1731230176671/Put/seqid=0 2024-11-10T09:16:17,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741852_1028 (size=12001) 2024-11-10T09:16:17,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:17,334 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/5d44ed13b7b5487a8f8b2958b9bf0ba3 2024-11-10T09:16:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/67b13496de7241c39059b22b1f091fe5 is 50, key is test_row_0/B:col10/1731230176671/Put/seqid=0 2024-11-10T09:16:17,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741853_1029 (size=12001) 2024-11-10T09:16:17,387 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/67b13496de7241c39059b22b1f091fe5 2024-11-10T09:16:17,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/5ef4d2b16ee94ed0b5727f1c979f763b is 50, key is test_row_0/C:col10/1731230176671/Put/seqid=0 2024-11-10T09:16:17,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230177363, max timestamp=1731230177401Num uniques 859; 2024-11-10T09:16:17,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:17,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230177363, max timestamp=1731230177401Num uniques 859; 2024-11-10T09:16:17,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:17,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230177363, max timestamp=1731230177401Num uniques 859; 2024-11-10T09:16:17,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:17,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741854_1030 (size=12001) 2024-11-10T09:16:17,446 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/5ef4d2b16ee94ed0b5727f1c979f763b 2024-11-10T09:16:17,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/5d44ed13b7b5487a8f8b2958b9bf0ba3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5d44ed13b7b5487a8f8b2958b9bf0ba3 2024-11-10T09:16:17,476 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5d44ed13b7b5487a8f8b2958b9bf0ba3, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T09:16:17,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/67b13496de7241c39059b22b1f091fe5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/67b13496de7241c39059b22b1f091fe5 2024-11-10T09:16:17,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230237466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230237466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230237470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,488 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T09:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,491 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/67b13496de7241c39059b22b1f091fe5, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T09:16:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230237471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/5ef4d2b16ee94ed0b5727f1c979f763b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5ef4d2b16ee94ed0b5727f1c979f763b 2024-11-10T09:16:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230237474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,505 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5ef4d2b16ee94ed0b5727f1c979f763b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T09:16:17,508 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for bfb4913e15c1cd99308737d49021ea5d in 231ms, sequenceid=82, compaction requested=true 2024-11-10T09:16:17,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T09:16:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T09:16:17,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T09:16:17,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6530 sec 2024-11-10T09:16:17,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6680 sec 2024-11-10T09:16:17,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-10T09:16:17,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:17,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/af70c510fbab450dac7b2843d31ba673 is 50, key is test_row_0/A:col10/1731230177465/Put/seqid=0 2024-11-10T09:16:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230237634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230237634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230237637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741855_1031 (size=12001) 2024-11-10T09:16:17,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230237642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230237643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230237748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230237748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230237748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230237758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230237758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230237968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230237968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230237969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230237968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230237969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:16:17,999 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:18,000 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T09:16:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:18,005 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:18,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:18,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:18,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/af70c510fbab450dac7b2843d31ba673 2024-11-10T09:16:18,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/12e7e50550e94de4abf8e13007d255c1 is 50, key is test_row_0/B:col10/1731230177465/Put/seqid=0 2024-11-10T09:16:18,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741856_1032 (size=12001) 2024-11-10T09:16:18,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/12e7e50550e94de4abf8e13007d255c1 2024-11-10T09:16:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:18,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/88bbb6af63004b09a849392d871932fe is 50, key is test_row_0/C:col10/1731230177465/Put/seqid=0 2024-11-10T09:16:18,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741857_1033 (size=12001) 2024-11-10T09:16:18,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/88bbb6af63004b09a849392d871932fe 2024-11-10T09:16:18,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:18,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/af70c510fbab450dac7b2843d31ba673 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/af70c510fbab450dac7b2843d31ba673 2024-11-10T09:16:18,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/af70c510fbab450dac7b2843d31ba673, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:16:18,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/12e7e50550e94de4abf8e13007d255c1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/12e7e50550e94de4abf8e13007d255c1 2024-11-10T09:16:18,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/12e7e50550e94de4abf8e13007d255c1, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:16:18,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/88bbb6af63004b09a849392d871932fe as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/88bbb6af63004b09a849392d871932fe 2024-11-10T09:16:18,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/88bbb6af63004b09a849392d871932fe, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:16:18,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=67.09 KB/68700 for bfb4913e15c1cd99308737d49021ea5d in 631ms, sequenceid=117, compaction requested=true 2024-11-10T09:16:18,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:18,231 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:18,231 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:18,240 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:18,240 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:18,240 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,240 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/596ccf60ac954237a4c1b422a77c6824, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bfb554bb12514c88a255cd1b1933b396, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5d44ed13b7b5487a8f8b2958b9bf0ba3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/af70c510fbab450dac7b2843d31ba673] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=51.5 K 2024-11-10T09:16:18,241 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596ccf60ac954237a4c1b422a77c6824, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731230175510 2024-11-10T09:16:18,242 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:18,242 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:18,242 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,242 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0d591b6007654381a9bc1151d3c44517, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/bb5cda50b9c54a739c53a055d575b691, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/67b13496de7241c39059b22b1f091fe5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/12e7e50550e94de4abf8e13007d255c1] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=47.0 K 2024-11-10T09:16:18,242 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfb554bb12514c88a255cd1b1933b396, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731230175909 2024-11-10T09:16:18,243 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d591b6007654381a9bc1151d3c44517, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731230175510 2024-11-10T09:16:18,243 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d44ed13b7b5487a8f8b2958b9bf0ba3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731230176664 2024-11-10T09:16:18,243 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5cda50b9c54a739c53a055d575b691, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731230175909 2024-11-10T09:16:18,244 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting af70c510fbab450dac7b2843d31ba673, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230177465 2024-11-10T09:16:18,245 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b13496de7241c39059b22b1f091fe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731230176664 2024-11-10T09:16:18,246 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e7e50550e94de4abf8e13007d255c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230177465 2024-11-10T09:16:18,278 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:18,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/1f708bde6b774a3fb0c2cb3840721f23 is 50, key is test_row_0/A:col10/1731230177465/Put/seqid=0 2024-11-10T09:16:18,288 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:18,289 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/0c51f8cbfcc541b7b91b232366cfd3ce is 50, key is test_row_0/B:col10/1731230177465/Put/seqid=0 2024-11-10T09:16:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:18,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T09:16:18,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:18,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741859_1035 (size=12241) 2024-11-10T09:16:18,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:18,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:18,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:18,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:18,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:18,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741858_1034 (size=12241) 2024-11-10T09:16:18,319 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/1f708bde6b774a3fb0c2cb3840721f23 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/1f708bde6b774a3fb0c2cb3840721f23 2024-11-10T09:16:18,319 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:16:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:18,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,325 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/0c51f8cbfcc541b7b91b232366cfd3ce as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0c51f8cbfcc541b7b91b232366cfd3ce 2024-11-10T09:16:18,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/5c68c2cdcfcb4f34a77e8f9db12bbc31 is 50, key is test_row_0/A:col10/1731230178293/Put/seqid=0 2024-11-10T09:16:18,345 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 1f708bde6b774a3fb0c2cb3840721f23(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:18,346 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:18,346 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=12, startTime=1731230178231; duration=0sec 2024-11-10T09:16:18,346 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:18,346 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:18,346 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:18,350 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:18,351 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:18,351 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,351 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/585639ad15ac443f8eb7b2bd34c003d1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/4bc38bde595947c2bdcf5c65b459483e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5ef4d2b16ee94ed0b5727f1c979f763b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/88bbb6af63004b09a849392d871932fe] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=47.0 K 2024-11-10T09:16:18,351 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 0c51f8cbfcc541b7b91b232366cfd3ce(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:18,351 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:18,351 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=12, startTime=1731230178231; duration=0sec 2024-11-10T09:16:18,352 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:18,352 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:18,352 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585639ad15ac443f8eb7b2bd34c003d1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731230175510 2024-11-10T09:16:18,353 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bc38bde595947c2bdcf5c65b459483e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731230175909 2024-11-10T09:16:18,354 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef4d2b16ee94ed0b5727f1c979f763b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731230176664 2024-11-10T09:16:18,355 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88bbb6af63004b09a849392d871932fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230177465 2024-11-10T09:16:18,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741860_1036 (size=12151) 2024-11-10T09:16:18,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/5c68c2cdcfcb4f34a77e8f9db12bbc31 2024-11-10T09:16:18,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230238348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,382 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#24 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:18,383 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/fe2b8754fb6b40ffb3647bbcda1b82e4 is 50, key is test_row_0/C:col10/1731230177465/Put/seqid=0 2024-11-10T09:16:18,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1e8f58a2af6c44f59440398ff774e308 is 50, key is test_row_0/B:col10/1731230178293/Put/seqid=0 2024-11-10T09:16:18,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230238357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230238363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230238377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230238389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741862_1038 (size=12151) 2024-11-10T09:16:18,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741861_1037 (size=12241) 2024-11-10T09:16:18,449 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/fe2b8754fb6b40ffb3647bbcda1b82e4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fe2b8754fb6b40ffb3647bbcda1b82e4 2024-11-10T09:16:18,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T09:16:18,454 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T09:16:18,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T09:16:18,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T09:16:18,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T09:16:18,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T09:16:18,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:16:18,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T09:16:18,468 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into fe2b8754fb6b40ffb3647bbcda1b82e4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:18,468 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:18,469 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=12, startTime=1731230178231; duration=0sec 2024-11-10T09:16:18,469 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:18,469 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:18,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230238487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230238498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230238498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230238514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230238514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:18,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:18,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:18,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230238697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230238718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230238727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230238727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230238727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:18,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1e8f58a2af6c44f59440398ff774e308 2024-11-10T09:16:18,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/38e6947bacf1455199532c7d0a93acfa is 50, key is test_row_0/C:col10/1731230178293/Put/seqid=0 2024-11-10T09:16:18,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741863_1039 (size=12151) 2024-11-10T09:16:18,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/38e6947bacf1455199532c7d0a93acfa 2024-11-10T09:16:18,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/5c68c2cdcfcb4f34a77e8f9db12bbc31 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5c68c2cdcfcb4f34a77e8f9db12bbc31 2024-11-10T09:16:18,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5c68c2cdcfcb4f34a77e8f9db12bbc31, entries=150, sequenceid=133, filesize=11.9 K 2024-11-10T09:16:18,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1e8f58a2af6c44f59440398ff774e308 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e8f58a2af6c44f59440398ff774e308 2024-11-10T09:16:18,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:18,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:18,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:18,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e8f58a2af6c44f59440398ff774e308, entries=150, sequenceid=133, filesize=11.9 K 2024-11-10T09:16:18,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/38e6947bacf1455199532c7d0a93acfa as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/38e6947bacf1455199532c7d0a93acfa 2024-11-10T09:16:18,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/38e6947bacf1455199532c7d0a93acfa, entries=150, sequenceid=133, filesize=11.9 K 2024-11-10T09:16:18,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for bfb4913e15c1cd99308737d49021ea5d in 670ms, sequenceid=133, compaction requested=false 2024-11-10T09:16:18,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:19,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:16:19,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:19,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:19,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:19,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:19,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1 is 50, key is test_row_0/A:col10/1731230178364/Put/seqid=0 2024-11-10T09:16:19,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230239062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230239062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230239065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230239071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741864_1040 (size=14541) 2024-11-10T09:16:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1 2024-11-10T09:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230239074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:19,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/4791314e30684fe4b9bd35d7ad9a61bb is 50, key is test_row_0/B:col10/1731230178364/Put/seqid=0 2024-11-10T09:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741865_1041 (size=12151) 2024-11-10T09:16:19,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/4791314e30684fe4b9bd35d7ad9a61bb 2024-11-10T09:16:19,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230239178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230239188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230239198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230239198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230239189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/bf1d0960d74a423bb8410c422ac604e1 is 50, key is test_row_0/C:col10/1731230178364/Put/seqid=0 2024-11-10T09:16:19,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741866_1042 (size=12151) 2024-11-10T09:16:19,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230239397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230239408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230239408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230239408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:19,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:19,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230239418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:19,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:19,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/bf1d0960d74a423bb8410c422ac604e1 2024-11-10T09:16:19,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1 2024-11-10T09:16:19,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1, entries=200, sequenceid=154, filesize=14.2 K 2024-11-10T09:16:19,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/4791314e30684fe4b9bd35d7ad9a61bb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/4791314e30684fe4b9bd35d7ad9a61bb 2024-11-10T09:16:19,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/4791314e30684fe4b9bd35d7ad9a61bb, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T09:16:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/bf1d0960d74a423bb8410c422ac604e1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bf1d0960d74a423bb8410c422ac604e1 2024-11-10T09:16:19,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bf1d0960d74a423bb8410c422ac604e1, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T09:16:19,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for bfb4913e15c1cd99308737d49021ea5d in 706ms, sequenceid=154, compaction requested=true 2024-11-10T09:16:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:19,723 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:19,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:19,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:19,723 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:19,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:19,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:19,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:19,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T09:16:19,725 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:19,725 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:19,725 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:19,725 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:19,725 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:19,725 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:19,725 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0c51f8cbfcc541b7b91b232366cfd3ce, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e8f58a2af6c44f59440398ff774e308, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/4791314e30684fe4b9bd35d7ad9a61bb] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=35.7 K 2024-11-10T09:16:19,725 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/1f708bde6b774a3fb0c2cb3840721f23, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5c68c2cdcfcb4f34a77e8f9db12bbc31, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=38.0 K 2024-11-10T09:16:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:19,727 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f708bde6b774a3fb0c2cb3840721f23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230177465 2024-11-10T09:16:19,727 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c51f8cbfcc541b7b91b232366cfd3ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230177465 2024-11-10T09:16:19,728 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c68c2cdcfcb4f34a77e8f9db12bbc31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230177637 2024-11-10T09:16:19,729 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfee37e2f9aa4fdbaa7b3a277ddaf8b1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230178353 2024-11-10T09:16:19,730 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e8f58a2af6c44f59440398ff774e308, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230177637 2024-11-10T09:16:19,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:19,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:19,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,732 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4791314e30684fe4b9bd35d7ad9a61bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230178353 2024-11-10T09:16:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/9d817c83d8804d2983d7417ce16fa436 is 50, key is test_row_0/A:col10/1731230179722/Put/seqid=0 2024-11-10T09:16:19,757 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:19,758 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/cc27a747617a40c6938deb63a3ef8e17 is 50, key is test_row_0/A:col10/1731230178364/Put/seqid=0 2024-11-10T09:16:19,775 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:19,776 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/c3da2b9c05fd4872b5edb0dbf78546bf is 50, key is test_row_0/B:col10/1731230178364/Put/seqid=0 2024-11-10T09:16:19,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230239784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230239785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230239786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230239788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230239796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741868_1044 (size=12493) 2024-11-10T09:16:19,834 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/cc27a747617a40c6938deb63a3ef8e17 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/cc27a747617a40c6938deb63a3ef8e17 2024-11-10T09:16:19,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741867_1043 (size=16931) 2024-11-10T09:16:19,847 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into cc27a747617a40c6938deb63a3ef8e17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:19,847 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:19,847 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230179722; duration=0sec 2024-11-10T09:16:19,848 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:19,848 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:19,848 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:19,850 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:19,850 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:19,850 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,850 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fe2b8754fb6b40ffb3647bbcda1b82e4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/38e6947bacf1455199532c7d0a93acfa, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bf1d0960d74a423bb8410c422ac604e1] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=35.7 K 2024-11-10T09:16:19,851 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe2b8754fb6b40ffb3647bbcda1b82e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230177465 2024-11-10T09:16:19,853 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38e6947bacf1455199532c7d0a93acfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230177637 2024-11-10T09:16:19,855 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1d0960d74a423bb8410c422ac604e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230178353 2024-11-10T09:16:19,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741869_1045 (size=12493) 2024-11-10T09:16:19,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:19,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:19,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:19,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:19,896 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/c3da2b9c05fd4872b5edb0dbf78546bf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c3da2b9c05fd4872b5edb0dbf78546bf 2024-11-10T09:16:19,901 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230239907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/e878c9edc1f044c8b42d2be70fccfa78 is 50, key is test_row_0/C:col10/1731230178364/Put/seqid=0 2024-11-10T09:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230239908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230239908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230239908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,918 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into c3da2b9c05fd4872b5edb0dbf78546bf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:19,918 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:19,918 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230179723; duration=0sec 2024-11-10T09:16:19,918 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:19,919 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230239918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:19,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741870_1046 (size=12493) 2024-11-10T09:16:19,963 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/e878c9edc1f044c8b42d2be70fccfa78 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e878c9edc1f044c8b42d2be70fccfa78 2024-11-10T09:16:19,977 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into e878c9edc1f044c8b42d2be70fccfa78(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:19,977 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:19,977 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230179723; duration=0sec 2024-11-10T09:16:19,978 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:19,978 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:20,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230240118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230240118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230240118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230240129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230240137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:20,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/9d817c83d8804d2983d7417ce16fa436 2024-11-10T09:16:20,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/aca1cd6a9987426b9c7ca49c1700bf90 is 50, key is test_row_0/B:col10/1731230179722/Put/seqid=0 2024-11-10T09:16:20,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741871_1047 (size=12151) 2024-11-10T09:16:20,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230240427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230240434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230240438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230240438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230240448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/aca1cd6a9987426b9c7ca49c1700bf90 2024-11-10T09:16:20,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/85df83e024e34ae39053b863b9170be8 is 50, key is test_row_0/C:col10/1731230179722/Put/seqid=0 2024-11-10T09:16:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741872_1048 (size=12151) 2024-11-10T09:16:20,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230240937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230240958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230240958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230240959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230240958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:20,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:21,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:21,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:21,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:21,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/85df83e024e34ae39053b863b9170be8 2024-11-10T09:16:21,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/9d817c83d8804d2983d7417ce16fa436 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/9d817c83d8804d2983d7417ce16fa436 2024-11-10T09:16:21,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/9d817c83d8804d2983d7417ce16fa436, entries=250, sequenceid=170, filesize=16.5 K 2024-11-10T09:16:21,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/aca1cd6a9987426b9c7ca49c1700bf90 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca1cd6a9987426b9c7ca49c1700bf90 2024-11-10T09:16:21,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca1cd6a9987426b9c7ca49c1700bf90, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T09:16:21,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/85df83e024e34ae39053b863b9170be8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/85df83e024e34ae39053b863b9170be8 2024-11-10T09:16:21,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/85df83e024e34ae39053b863b9170be8, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T09:16:21,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for bfb4913e15c1cd99308737d49021ea5d in 1518ms, sequenceid=170, compaction requested=false 2024-11-10T09:16:21,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:21,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:16:21,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:21,298 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T09:16:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/98d7c0a6363146a591355524f218c397 is 50, key is test_row_0/A:col10/1731230179785/Put/seqid=0 2024-11-10T09:16:21,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741873_1049 (size=12151) 2024-11-10T09:16:21,733 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/98d7c0a6363146a591355524f218c397 2024-11-10T09:16:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7800377cb4a44392bca20761954ae385 is 50, key is test_row_0/B:col10/1731230179785/Put/seqid=0 2024-11-10T09:16:21,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741874_1050 (size=12151) 2024-11-10T09:16:21,792 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7800377cb4a44392bca20761954ae385 2024-11-10T09:16:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/3374a8ac22004991872c04a4aeeceac0 is 50, key is test_row_0/C:col10/1731230179785/Put/seqid=0 2024-11-10T09:16:21,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741875_1051 (size=12151) 2024-11-10T09:16:21,841 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/3374a8ac22004991872c04a4aeeceac0 2024-11-10T09:16:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/98d7c0a6363146a591355524f218c397 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98d7c0a6363146a591355524f218c397 2024-11-10T09:16:21,864 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98d7c0a6363146a591355524f218c397, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T09:16:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7800377cb4a44392bca20761954ae385 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7800377cb4a44392bca20761954ae385 2024-11-10T09:16:21,884 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7800377cb4a44392bca20761954ae385, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T09:16:21,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/3374a8ac22004991872c04a4aeeceac0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/3374a8ac22004991872c04a4aeeceac0 2024-11-10T09:16:21,910 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/3374a8ac22004991872c04a4aeeceac0, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T09:16:21,912 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for bfb4913e15c1cd99308737d49021ea5d in 615ms, sequenceid=189, compaction requested=true 2024-11-10T09:16:21,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:21,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:21,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T09:16:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T09:16:21,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T09:16:21,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9070 sec 2024-11-10T09:16:21,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 3.9220 sec 2024-11-10T09:16:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:22,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:22,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:22,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:22,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:22,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:22,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:22,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:22,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8cea077cf9d84868ba890e4b416d7bb9 is 50, key is test_row_0/A:col10/1731230182002/Put/seqid=0 2024-11-10T09:16:22,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741876_1052 (size=12151) 2024-11-10T09:16:22,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230182129, max timestamp=1731230182143Num uniques 859; 2024-11-10T09:16:22,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:22,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230182129, max timestamp=1731230182143Num uniques 859; 2024-11-10T09:16:22,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:16:22,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:22,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T09:16:22,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230182129, max timestamp=1731230182143Num uniques 859; 2024-11-10T09:16:22,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:22,176 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:22,179 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:22,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:22,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230242225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230242228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230242230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230242230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230242233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:22,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:22,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230242337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230242349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230242369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230242370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230242370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8cea077cf9d84868ba890e4b416d7bb9 2024-11-10T09:16:22,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:22,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:22,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:22,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/5c07406c942e43e29691d09f81be5123 is 50, key is test_row_0/B:col10/1731230182002/Put/seqid=0 2024-11-10T09:16:22,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741877_1053 (size=12151) 2024-11-10T09:16:22,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/5c07406c942e43e29691d09f81be5123 2024-11-10T09:16:22,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/6c35adf79882445ca00a07d9cb11bf7f is 50, key is test_row_0/C:col10/1731230182002/Put/seqid=0 2024-11-10T09:16:22,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741878_1054 (size=12151) 2024-11-10T09:16:22,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/6c35adf79882445ca00a07d9cb11bf7f 2024-11-10T09:16:22,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230242568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230242568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8cea077cf9d84868ba890e4b416d7bb9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8cea077cf9d84868ba890e4b416d7bb9 2024-11-10T09:16:22,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8cea077cf9d84868ba890e4b416d7bb9, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T09:16:22,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/5c07406c942e43e29691d09f81be5123 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5c07406c942e43e29691d09f81be5123 2024-11-10T09:16:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230242588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5c07406c942e43e29691d09f81be5123, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T09:16:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/6c35adf79882445ca00a07d9cb11bf7f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6c35adf79882445ca00a07d9cb11bf7f 2024-11-10T09:16:22,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230242598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6c35adf79882445ca00a07d9cb11bf7f, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T09:16:22,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bfb4913e15c1cd99308737d49021ea5d in 605ms, sequenceid=200, compaction requested=true 2024-11-10T09:16:22,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:22,612 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:22,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:22,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:22,612 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:22,615 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:22,616 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:22,616 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,616 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/cc27a747617a40c6938deb63a3ef8e17, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/9d817c83d8804d2983d7417ce16fa436, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98d7c0a6363146a591355524f218c397, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8cea077cf9d84868ba890e4b416d7bb9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=52.5 K 2024-11-10T09:16:22,616 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:22,616 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:22,616 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,617 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c3da2b9c05fd4872b5edb0dbf78546bf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca1cd6a9987426b9c7ca49c1700bf90, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7800377cb4a44392bca20761954ae385, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5c07406c942e43e29691d09f81be5123] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=47.8 K 2024-11-10T09:16:22,618 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c3da2b9c05fd4872b5edb0dbf78546bf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230178353 2024-11-10T09:16:22,618 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc27a747617a40c6938deb63a3ef8e17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230178353 2024-11-10T09:16:22,619 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting aca1cd6a9987426b9c7ca49c1700bf90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230179058 2024-11-10T09:16:22,619 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d817c83d8804d2983d7417ce16fa436, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230179058 2024-11-10T09:16:22,620 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d7c0a6363146a591355524f218c397, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731230179783 2024-11-10T09:16:22,620 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7800377cb4a44392bca20761954ae385, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731230179783 2024-11-10T09:16:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:22,621 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cea077cf9d84868ba890e4b416d7bb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731230181989 2024-11-10T09:16:22,621 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c07406c942e43e29691d09f81be5123, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731230181989 2024-11-10T09:16:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:22,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=134.18 KB heapSize=385.22 KB 2024-11-10T09:16:22,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731230182225, max timestamp=1731230182632Num uniques 1018; 2024-11-10T09:16:22,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T09:16:22,663 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:22,663 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:22,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/52228b521c924ba2aa7ecf6d516200f7 is 50, key is test_row_0/A:col10/1731230182002/Put/seqid=0 2024-11-10T09:16:22,665 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/5f8a77d66efa4624a235ef6656910b1f is 50, key is test_row_0/B:col10/1731230182002/Put/seqid=0 2024-11-10T09:16:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:16:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:22,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:16:22,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/ce49e3c72be748d2aea3e34d6ec2f551 is 50, key is test_row_0/A:col10/1731230182632/Put/seqid=0 2024-11-10T09:16:22,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741880_1056 (size=12629) 2024-11-10T09:16:22,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741879_1055 (size=12629) 2024-11-10T09:16:22,781 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/5f8a77d66efa4624a235ef6656910b1f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5f8a77d66efa4624a235ef6656910b1f 2024-11-10T09:16:22,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741881_1057 (size=14541) 2024-11-10T09:16:22,794 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 5f8a77d66efa4624a235ef6656910b1f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:22,794 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:22,794 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=12, startTime=1731230182612; duration=0sec 2024-11-10T09:16:22,794 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:22,794 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:22,794 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:22,798 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:22,798 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:22,798 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,799 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e878c9edc1f044c8b42d2be70fccfa78, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/85df83e024e34ae39053b863b9170be8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/3374a8ac22004991872c04a4aeeceac0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6c35adf79882445ca00a07d9cb11bf7f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=47.8 K 2024-11-10T09:16:22,800 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e878c9edc1f044c8b42d2be70fccfa78, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230178353 2024-11-10T09:16:22,801 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 85df83e024e34ae39053b863b9170be8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230179058 2024-11-10T09:16:22,802 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 3374a8ac22004991872c04a4aeeceac0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731230179783 2024-11-10T09:16:22,804 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c35adf79882445ca00a07d9cb11bf7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731230181989 2024-11-10T09:16:22,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230242819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,836 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:22,837 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/b2f1c6ffd5ee4da197d7ef44000c161a is 50, key is test_row_0/C:col10/1731230182002/Put/seqid=0 2024-11-10T09:16:22,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230242878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230242896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741882_1058 (size=12629) 2024-11-10T09:16:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230242898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230242909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230242948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:22,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:22,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:22,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230243159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,165 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/52228b521c924ba2aa7ecf6d516200f7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/52228b521c924ba2aa7ecf6d516200f7 2024-11-10T09:16:23,180 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 52228b521c924ba2aa7ecf6d516200f7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:23,180 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:23,180 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=12, startTime=1731230182612; duration=0sec 2024-11-10T09:16:23,181 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:23,181 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:23,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/ce49e3c72be748d2aea3e34d6ec2f551 2024-11-10T09:16:23,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/9b95d891e24349afb932c23e9d4ee466 is 50, key is test_row_0/B:col10/1731230182632/Put/seqid=0 2024-11-10T09:16:23,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741883_1059 (size=12151) 2024-11-10T09:16:23,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/9b95d891e24349afb932c23e9d4ee466 2024-11-10T09:16:23,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/7d9b8606e6514296808457c6b5957f70 is 50, key is test_row_0/C:col10/1731230182632/Put/seqid=0 2024-11-10T09:16:23,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741884_1060 (size=12151) 2024-11-10T09:16:23,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/7d9b8606e6514296808457c6b5957f70 2024-11-10T09:16:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/ce49e3c72be748d2aea3e34d6ec2f551 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/ce49e3c72be748d2aea3e34d6ec2f551 2024-11-10T09:16:23,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/ce49e3c72be748d2aea3e34d6ec2f551, entries=200, sequenceid=238, filesize=14.2 K 2024-11-10T09:16:23,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/9b95d891e24349afb932c23e9d4ee466 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/9b95d891e24349afb932c23e9d4ee466 2024-11-10T09:16:23,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/9b95d891e24349afb932c23e9d4ee466, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T09:16:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/7d9b8606e6514296808457c6b5957f70 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/7d9b8606e6514296808457c6b5957f70 2024-11-10T09:16:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,312 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/b2f1c6ffd5ee4da197d7ef44000c161a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/b2f1c6ffd5ee4da197d7ef44000c161a 2024-11-10T09:16:23,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/7d9b8606e6514296808457c6b5957f70, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T09:16:23,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for bfb4913e15c1cd99308737d49021ea5d in 676ms, sequenceid=238, compaction requested=false 2024-11-10T09:16:23,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:23,329 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into b2f1c6ffd5ee4da197d7ef44000c161a(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:23,329 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:23,329 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=12, startTime=1731230182638; duration=0sec 2024-11-10T09:16:23,329 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:23,329 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:23,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T09:16:23,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:23,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:23,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:23,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:23,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:23,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:23,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/da0fe346617948c2ab81583a8394a9b2 is 50, key is test_row_0/A:col10/1731230182811/Put/seqid=0 2024-11-10T09:16:23,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230243446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230243447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230243449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230243454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230243468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230243557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230243569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230243570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230243571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230243767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741885_1061 (size=14541) 2024-11-10T09:16:23,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230243778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230243779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230243778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:23,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:23,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:23,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:23,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230243983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:24,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:24,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:24,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:24,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230244077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230244088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230244088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230244100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/da0fe346617948c2ab81583a8394a9b2 2024-11-10T09:16:24,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/c17662004f274219acaa7a857cf97000 is 50, key is test_row_0/B:col10/1731230182811/Put/seqid=0 2024-11-10T09:16:24,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:24,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741886_1062 (size=12151) 2024-11-10T09:16:24,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/c17662004f274219acaa7a857cf97000 2024-11-10T09:16:24,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/bce74014ea76462e8fc98ed43072e0e9 is 50, key is test_row_0/C:col10/1731230182811/Put/seqid=0 2024-11-10T09:16:24,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741887_1063 (size=12151) 2024-11-10T09:16:24,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/bce74014ea76462e8fc98ed43072e0e9 2024-11-10T09:16:24,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/da0fe346617948c2ab81583a8394a9b2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/da0fe346617948c2ab81583a8394a9b2 2024-11-10T09:16:24,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/da0fe346617948c2ab81583a8394a9b2, entries=200, sequenceid=256, filesize=14.2 K 2024-11-10T09:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:24,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/c17662004f274219acaa7a857cf97000 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c17662004f274219acaa7a857cf97000 2024-11-10T09:16:24,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c17662004f274219acaa7a857cf97000, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T09:16:24,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/bce74014ea76462e8fc98ed43072e0e9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bce74014ea76462e8fc98ed43072e0e9 2024-11-10T09:16:24,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bce74014ea76462e8fc98ed43072e0e9, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T09:16:24,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for bfb4913e15c1cd99308737d49021ea5d in 970ms, sequenceid=256, compaction requested=true 2024-11-10T09:16:24,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:24,372 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:24,372 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:24,373 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:24,373 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:24,374 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:24,374 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:24,374 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,374 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,374 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/52228b521c924ba2aa7ecf6d516200f7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/ce49e3c72be748d2aea3e34d6ec2f551, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/da0fe346617948c2ab81583a8394a9b2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=40.7 K 2024-11-10T09:16:24,374 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5f8a77d66efa4624a235ef6656910b1f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/9b95d891e24349afb932c23e9d4ee466, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c17662004f274219acaa7a857cf97000] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.1 K 2024-11-10T09:16:24,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52228b521c924ba2aa7ecf6d516200f7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731230181989 2024-11-10T09:16:24,375 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8a77d66efa4624a235ef6656910b1f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731230181989 2024-11-10T09:16:24,375 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b95d891e24349afb932c23e9d4ee466, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731230182225 2024-11-10T09:16:24,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce49e3c72be748d2aea3e34d6ec2f551, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731230182225 2024-11-10T09:16:24,376 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0fe346617948c2ab81583a8394a9b2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731230182728 2024-11-10T09:16:24,376 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c17662004f274219acaa7a857cf97000, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731230182781 2024-11-10T09:16:24,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:16:24,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,401 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T09:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:24,410 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:24,413 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8a36d31fbc654d35aaded27804647347 is 50, key is test_row_0/A:col10/1731230182811/Put/seqid=0 2024-11-10T09:16:24,422 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:24,423 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/2372d2f715034d4cbda61057acee8c62 is 50, key is test_row_0/B:col10/1731230182811/Put/seqid=0 2024-11-10T09:16:24,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/0dfcfa349f98458b8900038fde8707d9 is 50, key is test_row_0/A:col10/1731230183447/Put/seqid=0 2024-11-10T09:16:24,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741889_1065 (size=12731) 2024-11-10T09:16:24,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741888_1064 (size=12731) 2024-11-10T09:16:24,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741890_1066 (size=12301) 2024-11-10T09:16:24,463 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/0dfcfa349f98458b8900038fde8707d9 2024-11-10T09:16:24,465 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/2372d2f715034d4cbda61057acee8c62 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/2372d2f715034d4cbda61057acee8c62 2024-11-10T09:16:24,500 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 2372d2f715034d4cbda61057acee8c62(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:24,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:24,500 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230184372; duration=0sec 2024-11-10T09:16:24,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:24,501 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:24,501 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:24,505 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:24,505 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:24,505 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:24,506 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8a36d31fbc654d35aaded27804647347 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8a36d31fbc654d35aaded27804647347 2024-11-10T09:16:24,506 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/b2f1c6ffd5ee4da197d7ef44000c161a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/7d9b8606e6514296808457c6b5957f70, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bce74014ea76462e8fc98ed43072e0e9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.1 K 2024-11-10T09:16:24,507 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f1c6ffd5ee4da197d7ef44000c161a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731230181989 2024-11-10T09:16:24,507 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d9b8606e6514296808457c6b5957f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731230182225 2024-11-10T09:16:24,508 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting bce74014ea76462e8fc98ed43072e0e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731230182781 2024-11-10T09:16:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/af495715bc824daabe00b3537b1f7ccd is 50, key is test_row_0/B:col10/1731230183447/Put/seqid=0 2024-11-10T09:16:24,519 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 8a36d31fbc654d35aaded27804647347(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:24,519 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:24,519 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230184371; duration=0sec 2024-11-10T09:16:24,519 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:24,519 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:24,549 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:24,553 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/90e96d7a36dc4bbf8153833f2243b1ca is 50, key is test_row_0/C:col10/1731230182811/Put/seqid=0 2024-11-10T09:16:24,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741891_1067 (size=12301) 2024-11-10T09:16:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:24,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:24,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741892_1068 (size=12731) 2024-11-10T09:16:24,654 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/90e96d7a36dc4bbf8153833f2243b1ca as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/90e96d7a36dc4bbf8153833f2243b1ca 2024-11-10T09:16:24,680 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into 90e96d7a36dc4bbf8153833f2243b1ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:24,681 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:24,681 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230184372; duration=0sec 2024-11-10T09:16:24,681 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:24,681 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:24,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230244684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230244687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230244690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230244691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230244808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230244808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230244818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230244820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:24,996 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/af495715bc824daabe00b3537b1f7ccd 2024-11-10T09:16:25,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230245000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/29ee56619ced4a59b4ade67ccec6ca69 is 50, key is test_row_0/C:col10/1731230183447/Put/seqid=0 2024-11-10T09:16:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230245019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,020 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in bfb4913e15c1cd99308737d49021ea5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:16:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230245027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230245027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230245028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741893_1069 (size=12301) 2024-11-10T09:16:25,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230245340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230245349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230245352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230245349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,490 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/29ee56619ced4a59b4ade67ccec6ca69 2024-11-10T09:16:25,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/0dfcfa349f98458b8900038fde8707d9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/0dfcfa349f98458b8900038fde8707d9 2024-11-10T09:16:25,543 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/0dfcfa349f98458b8900038fde8707d9, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T09:16:25,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/af495715bc824daabe00b3537b1f7ccd as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/af495715bc824daabe00b3537b1f7ccd 2024-11-10T09:16:25,556 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/af495715bc824daabe00b3537b1f7ccd, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T09:16:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/29ee56619ced4a59b4ade67ccec6ca69 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/29ee56619ced4a59b4ade67ccec6ca69 2024-11-10T09:16:25,577 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/29ee56619ced4a59b4ade67ccec6ca69, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T09:16:25,580 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for bfb4913e15c1cd99308737d49021ea5d in 1179ms, sequenceid=273, compaction requested=false 2024-11-10T09:16:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T09:16:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T09:16:25,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T09:16:25,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4050 sec 2024-11-10T09:16:25,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 3.4190 sec 2024-11-10T09:16:25,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T09:16:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:25,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/523f85bf8f01494e923bc625ddc67ff4 is 50, key is test_row_0/A:col10/1731230184667/Put/seqid=0 2024-11-10T09:16:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230245907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230245909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230245912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230245918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:25,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741894_1070 (size=14741) 2024-11-10T09:16:25,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/523f85bf8f01494e923bc625ddc67ff4 2024-11-10T09:16:25,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/991b07213cf343ca9fc9095d9170eec8 is 50, key is test_row_0/B:col10/1731230184667/Put/seqid=0 2024-11-10T09:16:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230246028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230246028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230246028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741895_1071 (size=12301) 2024-11-10T09:16:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230246038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230246248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230246249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230246261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230246261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:16:26,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:26,339 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T09:16:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:16:26,356 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:26,360 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:26,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:26,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/991b07213cf343ca9fc9095d9170eec8 2024-11-10T09:16:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:16:26,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/43646280b52a45468938001859c786d6 is 50, key is test_row_0/C:col10/1731230184667/Put/seqid=0 2024-11-10T09:16:26,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:16:26,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:26,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:26,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:26,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:26,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:26,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741896_1072 (size=12301) 2024-11-10T09:16:26,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/43646280b52a45468938001859c786d6 2024-11-10T09:16:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230246568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230246568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/523f85bf8f01494e923bc625ddc67ff4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/523f85bf8f01494e923bc625ddc67ff4 2024-11-10T09:16:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230246588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230246591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:26,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/523f85bf8f01494e923bc625ddc67ff4, entries=200, sequenceid=293, filesize=14.4 K 2024-11-10T09:16:26,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/991b07213cf343ca9fc9095d9170eec8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/991b07213cf343ca9fc9095d9170eec8 2024-11-10T09:16:26,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/991b07213cf343ca9fc9095d9170eec8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T09:16:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/43646280b52a45468938001859c786d6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/43646280b52a45468938001859c786d6 2024-11-10T09:16:26,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/43646280b52a45468938001859c786d6, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T09:16:26,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for bfb4913e15c1cd99308737d49021ea5d in 771ms, sequenceid=293, compaction requested=true 2024-11-10T09:16:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:26,638 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:26,639 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:26,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:16:26,640 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:26,640 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:26,640 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:26,641 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8a36d31fbc654d35aaded27804647347, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/0dfcfa349f98458b8900038fde8707d9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/523f85bf8f01494e923bc625ddc67ff4] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=38.8 K 2024-11-10T09:16:26,644 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a36d31fbc654d35aaded27804647347, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731230182781 2024-11-10T09:16:26,644 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:26,645 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:26,645 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:26,645 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/2372d2f715034d4cbda61057acee8c62, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/af495715bc824daabe00b3537b1f7ccd, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/991b07213cf343ca9fc9095d9170eec8] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.5 K 2024-11-10T09:16:26,646 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 2372d2f715034d4cbda61057acee8c62, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731230182781 2024-11-10T09:16:26,646 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dfcfa349f98458b8900038fde8707d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731230183439 2024-11-10T09:16:26,646 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting af495715bc824daabe00b3537b1f7ccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731230183439 2024-11-10T09:16:26,647 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523f85bf8f01494e923bc625ddc67ff4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731230184646 2024-11-10T09:16:26,648 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 991b07213cf343ca9fc9095d9170eec8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731230184667 2024-11-10T09:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:16:26,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:16:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:26,680 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T09:16:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:26,687 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:26,688 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/771a5b1a72b5493a9638c95a674640a3 is 50, key is test_row_0/B:col10/1731230184667/Put/seqid=0 2024-11-10T09:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/e3c8508c05c9466095875de8eb246695 is 50, key is test_row_0/A:col10/1731230185913/Put/seqid=0 2024-11-10T09:16:26,712 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:26,713 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/d3aa287b7b224f5a8fdffebc959ca106 is 50, key is test_row_0/A:col10/1731230184667/Put/seqid=0 2024-11-10T09:16:26,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741897_1073 (size=12983) 2024-11-10T09:16:26,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741899_1075 (size=12983) 2024-11-10T09:16:26,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741898_1074 (size=12301) 2024-11-10T09:16:26,747 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/e3c8508c05c9466095875de8eb246695 2024-11-10T09:16:26,748 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/771a5b1a72b5493a9638c95a674640a3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/771a5b1a72b5493a9638c95a674640a3 2024-11-10T09:16:26,780 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/d3aa287b7b224f5a8fdffebc959ca106 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/d3aa287b7b224f5a8fdffebc959ca106 2024-11-10T09:16:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/6ab3862015e54debb5669f1ae25e6e40 is 50, key is test_row_0/B:col10/1731230185913/Put/seqid=0 2024-11-10T09:16:26,796 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 771a5b1a72b5493a9638c95a674640a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:26,796 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:26,796 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230186639; duration=0sec 2024-11-10T09:16:26,796 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:26,796 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:26,797 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:26,803 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:26,803 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:26,804 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:26,804 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/90e96d7a36dc4bbf8153833f2243b1ca, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/29ee56619ced4a59b4ade67ccec6ca69, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/43646280b52a45468938001859c786d6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.5 K 2024-11-10T09:16:26,804 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into d3aa287b7b224f5a8fdffebc959ca106(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:26,804 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:26,804 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230186638; duration=0sec 2024-11-10T09:16:26,805 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e96d7a36dc4bbf8153833f2243b1ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731230182781 2024-11-10T09:16:26,805 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:26,805 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:26,805 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ee56619ced4a59b4ade67ccec6ca69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731230183439 2024-11-10T09:16:26,806 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 43646280b52a45468938001859c786d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731230184667 2024-11-10T09:16:26,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741900_1076 (size=12301) 2024-11-10T09:16:26,836 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/6ab3862015e54debb5669f1ae25e6e40 2024-11-10T09:16:26,838 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:26,839 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/0eb05834e7aa4c4587a9f9fb1f5b20a9 is 50, key is test_row_0/C:col10/1731230184667/Put/seqid=0 2024-11-10T09:16:26,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741901_1077 (size=12983) 2024-11-10T09:16:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/fcb843e46fb2434897678da7a8aeabee is 50, key is test_row_0/C:col10/1731230185913/Put/seqid=0 2024-11-10T09:16:26,889 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/0eb05834e7aa4c4587a9f9fb1f5b20a9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/0eb05834e7aa4c4587a9f9fb1f5b20a9 2024-11-10T09:16:26,941 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into 0eb05834e7aa4c4587a9f9fb1f5b20a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:26,941 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:26,941 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230186639; duration=0sec 2024-11-10T09:16:26,942 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:26,942 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:26,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741902_1078 (size=12301) 2024-11-10T09:16:26,952 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/fcb843e46fb2434897678da7a8aeabee 2024-11-10T09:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:16:27,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/e3c8508c05c9466095875de8eb246695 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/e3c8508c05c9466095875de8eb246695 2024-11-10T09:16:27,026 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/e3c8508c05c9466095875de8eb246695, entries=150, sequenceid=308, filesize=12.0 K 2024-11-10T09:16:27,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/6ab3862015e54debb5669f1ae25e6e40 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6ab3862015e54debb5669f1ae25e6e40 2024-11-10T09:16:27,050 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6ab3862015e54debb5669f1ae25e6e40, entries=150, sequenceid=308, filesize=12.0 K 2024-11-10T09:16:27,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/fcb843e46fb2434897678da7a8aeabee as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fcb843e46fb2434897678da7a8aeabee 2024-11-10T09:16:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:27,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:27,079 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fcb843e46fb2434897678da7a8aeabee, entries=150, sequenceid=308, filesize=12.0 K 2024-11-10T09:16:27,082 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for bfb4913e15c1cd99308737d49021ea5d in 401ms, sequenceid=308, compaction requested=false 2024-11-10T09:16:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T09:16:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T09:16:27,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T09:16:27,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 724 msec 2024-11-10T09:16:27,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 749 msec 2024-11-10T09:16:27,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:27,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/98589f40cdc645129141c23623839534 is 50, key is test_row_0/A:col10/1731230187085/Put/seqid=0 2024-11-10T09:16:27,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741903_1079 (size=12301) 2024-11-10T09:16:27,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230187148, max timestamp=1731230187184Num uniques 859; 2024-11-10T09:16:27,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230187148, max timestamp=1731230187184Num uniques 859; 2024-11-10T09:16:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:16:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:16:27,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230187148, max timestamp=1731230187180Num uniques 859; 2024-11-10T09:16:27,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:27,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/98589f40cdc645129141c23623839534 2024-11-10T09:16:27,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/29a7dcce2f8a4d91a31df791fdd93562 is 50, key is test_row_0/B:col10/1731230187085/Put/seqid=0 2024-11-10T09:16:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741904_1080 (size=12301) 2024-11-10T09:16:27,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230247259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230247262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230247271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230247275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230247275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230247382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230247382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230247388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230247390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230247390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:16:27,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:27,489 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T09:16:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:16:27,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:27,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:27,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:16:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230247601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230247601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230247608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230247608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230247608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T09:16:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/29a7dcce2f8a4d91a31df791fdd93562 2024-11-10T09:16:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/42b4b13592d34d90b58adbec2db02f46 is 50, key is test_row_0/C:col10/1731230187085/Put/seqid=0 2024-11-10T09:16:27,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741905_1081 (size=12301) 2024-11-10T09:16:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:16:27,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T09:16:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230247922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230247930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230247930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230247932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230247932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:27,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:27,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:27,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T09:16:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:27,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:28,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/42b4b13592d34d90b58adbec2db02f46 2024-11-10T09:16:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/98589f40cdc645129141c23623839534 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98589f40cdc645129141c23623839534 2024-11-10T09:16:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:16:28,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98589f40cdc645129141c23623839534, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T09:16:28,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/29a7dcce2f8a4d91a31df791fdd93562 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/29a7dcce2f8a4d91a31df791fdd93562 2024-11-10T09:16:28,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/29a7dcce2f8a4d91a31df791fdd93562, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T09:16:28,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T09:16:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/42b4b13592d34d90b58adbec2db02f46 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/42b4b13592d34d90b58adbec2db02f46 2024-11-10T09:16:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:28,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/42b4b13592d34d90b58adbec2db02f46, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T09:16:28,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=143.13 KB/146560 for bfb4913e15c1cd99308737d49021ea5d in 1070ms, sequenceid=321, compaction requested=true 2024-11-10T09:16:28,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:28,176 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:28,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:28,177 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:28,177 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:28,177 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,178 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/d3aa287b7b224f5a8fdffebc959ca106, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/e3c8508c05c9466095875de8eb246695, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98589f40cdc645129141c23623839534] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.7 K 2024-11-10T09:16:28,182 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3aa287b7b224f5a8fdffebc959ca106, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731230184667 2024-11-10T09:16:28,183 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c8508c05c9466095875de8eb246695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731230185896 2024-11-10T09:16:28,183 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98589f40cdc645129141c23623839534, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230187085 2024-11-10T09:16:28,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:28,184 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:28,186 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:28,186 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:28,186 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,186 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/771a5b1a72b5493a9638c95a674640a3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6ab3862015e54debb5669f1ae25e6e40, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/29a7dcce2f8a4d91a31df791fdd93562] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.7 K 2024-11-10T09:16:28,187 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 771a5b1a72b5493a9638c95a674640a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731230184667 2024-11-10T09:16:28,188 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab3862015e54debb5669f1ae25e6e40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731230185896 2024-11-10T09:16:28,189 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a7dcce2f8a4d91a31df791fdd93562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230187085 2024-11-10T09:16:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:28,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:28,203 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:28,204 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/34411a0630354a8dbe722dd1dc10bfc9 is 50, key is test_row_0/A:col10/1731230187085/Put/seqid=0 2024-11-10T09:16:28,216 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:28,217 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/edb4aabb7568477cbcaa48c717b010ba is 50, key is test_row_0/B:col10/1731230187085/Put/seqid=0 2024-11-10T09:16:28,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741906_1082 (size=13085) 2024-11-10T09:16:28,257 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/34411a0630354a8dbe722dd1dc10bfc9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/34411a0630354a8dbe722dd1dc10bfc9 2024-11-10T09:16:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741907_1083 (size=13085) 2024-11-10T09:16:28,277 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 34411a0630354a8dbe722dd1dc10bfc9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:28,277 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:28,277 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230188176; duration=0sec 2024-11-10T09:16:28,278 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:28,278 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:28,278 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:28,289 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:28,289 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:28,289 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,289 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/0eb05834e7aa4c4587a9f9fb1f5b20a9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fcb843e46fb2434897678da7a8aeabee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/42b4b13592d34d90b58adbec2db02f46] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.7 K 2024-11-10T09:16:28,290 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb05834e7aa4c4587a9f9fb1f5b20a9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731230184667 2024-11-10T09:16:28,290 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb843e46fb2434897678da7a8aeabee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731230185896 2024-11-10T09:16:28,291 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b4b13592d34d90b58adbec2db02f46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230187085 2024-11-10T09:16:28,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T09:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,302 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=143.13 KB heapSize=405.30 KB 2024-11-10T09:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:28,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:28,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/785613e214b341f586f131e77c86ad8a is 50, key is test_row_0/A:col10/1731230187268/Put/seqid=0 2024-11-10T09:16:28,333 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:28,334 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/5df41a5cb52f4cce8353895811236cdc is 50, key is test_row_0/C:col10/1731230187085/Put/seqid=0 2024-11-10T09:16:28,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741908_1084 (size=12301) 2024-11-10T09:16:28,354 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/785613e214b341f586f131e77c86ad8a 2024-11-10T09:16:28,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741909_1085 (size=13085) 2024-11-10T09:16:28,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1c2e9416d447435d8563540c6fe0f87e is 50, key is test_row_0/B:col10/1731230187268/Put/seqid=0 2024-11-10T09:16:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741910_1086 (size=12301) 2024-11-10T09:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:28,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230248451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230248451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230248453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230248453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230248457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230248568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230248568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230248568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230248568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:16:28,682 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/edb4aabb7568477cbcaa48c717b010ba as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/edb4aabb7568477cbcaa48c717b010ba 2024-11-10T09:16:28,693 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into edb4aabb7568477cbcaa48c717b010ba(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:28,693 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:28,693 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230188184; duration=0sec 2024-11-10T09:16:28,693 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:28,693 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:28,791 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/5df41a5cb52f4cce8353895811236cdc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5df41a5cb52f4cce8353895811236cdc 2024-11-10T09:16:28,795 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1c2e9416d447435d8563540c6fe0f87e 2024-11-10T09:16:28,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230248787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230248787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230248787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230248788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:28,811 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into 5df41a5cb52f4cce8353895811236cdc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:28,811 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:28,811 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230188189; duration=0sec 2024-11-10T09:16:28,811 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:28,811 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/289a56a83cc84676a21373ad02f47914 is 50, key is test_row_0/C:col10/1731230187268/Put/seqid=0 2024-11-10T09:16:28,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741911_1087 (size=12301) 2024-11-10T09:16:28,844 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/289a56a83cc84676a21373ad02f47914 2024-11-10T09:16:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/785613e214b341f586f131e77c86ad8a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/785613e214b341f586f131e77c86ad8a 2024-11-10T09:16:28,871 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/785613e214b341f586f131e77c86ad8a, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T09:16:28,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1c2e9416d447435d8563540c6fe0f87e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1c2e9416d447435d8563540c6fe0f87e 2024-11-10T09:16:28,887 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1c2e9416d447435d8563540c6fe0f87e, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T09:16:28,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/289a56a83cc84676a21373ad02f47914 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/289a56a83cc84676a21373ad02f47914 2024-11-10T09:16:28,907 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/289a56a83cc84676a21373ad02f47914, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T09:16:28,908 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~405.25 KB/414976, currentSize=40.25 KB/41220 for bfb4913e15c1cd99308737d49021ea5d in 606ms, sequenceid=359, compaction requested=false 2024-11-10T09:16:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T09:16:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T09:16:28,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T09:16:28,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-10T09:16:28,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4260 sec 2024-11-10T09:16:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/3e82c1c911984f02a3d1e51064a9be93 is 50, key is test_row_0/A:col10/1731230189111/Put/seqid=0 2024-11-10T09:16:29,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741912_1088 (size=12297) 2024-11-10T09:16:29,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230189158, max timestamp=1731230189192Num uniques 859; 2024-11-10T09:16:29,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:29,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230189158, max timestamp=1731230189192Num uniques 859; 2024-11-10T09:16:29,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:16:29,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230189158, max timestamp=1731230189192Num uniques 859; 2024-11-10T09:16:29,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:16:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230249265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230249265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230249273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230249271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230249391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230249391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230249397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230249398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230249478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,482 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in bfb4913e15c1cd99308737d49021ea5d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:16:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/3e82c1c911984f02a3d1e51064a9be93 2024-11-10T09:16:29,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1e0259eeea734b2c96526645ff81bc9d is 50, key is test_row_0/B:col10/1731230189111/Put/seqid=0 2024-11-10T09:16:29,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230249612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230249618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230249619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230249619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:16:29,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:29,649 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T09:16:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:16:29,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:29,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:29,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:29,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741913_1089 (size=9857) 2024-11-10T09:16:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:16:29,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:16:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:29,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230249927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230249928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230249938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230249938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:29,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:29,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:29,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:16:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:29,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:16:30,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1e0259eeea734b2c96526645ff81bc9d 2024-11-10T09:16:30,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/1f9712b59d46493391afccdfbc2d6682 is 50, key is test_row_0/C:col10/1731230189111/Put/seqid=0 2024-11-10T09:16:30,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:16:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741914_1090 (size=9857) 2024-11-10T09:16:30,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:16:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:16:30,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230250437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:30,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:16:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:30,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230250448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:30,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230250448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:30,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230250448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:30,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/1f9712b59d46493391afccdfbc2d6682 2024-11-10T09:16:30,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/3e82c1c911984f02a3d1e51064a9be93 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/3e82c1c911984f02a3d1e51064a9be93 2024-11-10T09:16:30,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/3e82c1c911984f02a3d1e51064a9be93, entries=150, sequenceid=372, filesize=12.0 K 2024-11-10T09:16:30,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1e0259eeea734b2c96526645ff81bc9d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e0259eeea734b2c96526645ff81bc9d 2024-11-10T09:16:30,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e0259eeea734b2c96526645ff81bc9d, entries=100, sequenceid=372, filesize=9.6 K 2024-11-10T09:16:30,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/1f9712b59d46493391afccdfbc2d6682 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f9712b59d46493391afccdfbc2d6682 2024-11-10T09:16:30,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f9712b59d46493391afccdfbc2d6682, entries=100, sequenceid=372, filesize=9.6 K 2024-11-10T09:16:30,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for bfb4913e15c1cd99308737d49021ea5d in 1486ms, sequenceid=372, compaction requested=true 2024-11-10T09:16:30,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:30,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:30,602 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:30,602 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:30,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:30,603 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:30,603 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:30,603 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,603 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/34411a0630354a8dbe722dd1dc10bfc9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/785613e214b341f586f131e77c86ad8a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/3e82c1c911984f02a3d1e51064a9be93] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.8 K 2024-11-10T09:16:30,604 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34411a0630354a8dbe722dd1dc10bfc9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230187085 2024-11-10T09:16:30,604 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:30,604 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 785613e214b341f586f131e77c86ad8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731230187267 2024-11-10T09:16:30,605 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:30,605 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,605 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/edb4aabb7568477cbcaa48c717b010ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1c2e9416d447435d8563540c6fe0f87e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e0259eeea734b2c96526645ff81bc9d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=34.4 K 2024-11-10T09:16:30,605 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e82c1c911984f02a3d1e51064a9be93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731230188451 2024-11-10T09:16:30,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting edb4aabb7568477cbcaa48c717b010ba, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230187085 2024-11-10T09:16:30,607 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2e9416d447435d8563540c6fe0f87e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731230187267 2024-11-10T09:16:30,608 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0259eeea734b2c96526645ff81bc9d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731230188451 2024-11-10T09:16:30,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:16:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,613 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=126.68 KB heapSize=363.23 KB 2024-11-10T09:16:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:30,622 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:30,623 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/4604c155c6f94744966e49b71e7929fb is 50, key is test_row_0/A:col10/1731230189111/Put/seqid=0 2024-11-10T09:16:30,631 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:30,631 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/ee81ca86628f45a29dbb674a0f1a3f67 is 50, key is test_row_0/B:col10/1731230189111/Put/seqid=0 2024-11-10T09:16:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8248691b1dc441a48fdaadafd6bac160 is 50, key is test_row_0/A:col10/1731230189270/Put/seqid=0 2024-11-10T09:16:30,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741915_1091 (size=13187) 2024-11-10T09:16:30,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741916_1092 (size=13187) 2024-11-10T09:16:30,666 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/4604c155c6f94744966e49b71e7929fb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/4604c155c6f94744966e49b71e7929fb 2024-11-10T09:16:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741917_1093 (size=12301) 2024-11-10T09:16:30,681 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8248691b1dc441a48fdaadafd6bac160 2024-11-10T09:16:30,690 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 4604c155c6f94744966e49b71e7929fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:30,690 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:30,690 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230190601; duration=0sec 2024-11-10T09:16:30,691 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/ee81ca86628f45a29dbb674a0f1a3f67 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/ee81ca86628f45a29dbb674a0f1a3f67 2024-11-10T09:16:30,691 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:30,691 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:30,691 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:30,696 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:30,696 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:30,696 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,698 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5df41a5cb52f4cce8353895811236cdc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/289a56a83cc84676a21373ad02f47914, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f9712b59d46493391afccdfbc2d6682] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=34.4 K 2024-11-10T09:16:30,700 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df41a5cb52f4cce8353895811236cdc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230187085 2024-11-10T09:16:30,702 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289a56a83cc84676a21373ad02f47914, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731230187267 2024-11-10T09:16:30,705 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9712b59d46493391afccdfbc2d6682, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731230188451 2024-11-10T09:16:30,708 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into ee81ca86628f45a29dbb674a0f1a3f67(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:30,708 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:30,708 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230190602; duration=0sec 2024-11-10T09:16:30,708 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:30,708 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:30,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7acca5a141774323a470ee4df89b65b7 is 50, key is test_row_0/B:col10/1731230189270/Put/seqid=0 2024-11-10T09:16:30,727 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:30,727 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/a571df21cff34e36857ee248c17948f2 is 50, key is test_row_0/C:col10/1731230189111/Put/seqid=0 2024-11-10T09:16:30,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741918_1094 (size=12301) 2024-11-10T09:16:30,747 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7acca5a141774323a470ee4df89b65b7 2024-11-10T09:16:30,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741919_1095 (size=13187) 2024-11-10T09:16:30,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/6d889531306a48f09e496b58c5cc4555 is 50, key is test_row_0/C:col10/1731230189270/Put/seqid=0 2024-11-10T09:16:30,775 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/a571df21cff34e36857ee248c17948f2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/a571df21cff34e36857ee248c17948f2 2024-11-10T09:16:30,789 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into a571df21cff34e36857ee248c17948f2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:30,789 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:30,789 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230190602; duration=0sec 2024-11-10T09:16:30,789 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:30,789 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:16:30,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741920_1096 (size=12301) 2024-11-10T09:16:30,801 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/6d889531306a48f09e496b58c5cc4555 2024-11-10T09:16:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/8248691b1dc441a48fdaadafd6bac160 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8248691b1dc441a48fdaadafd6bac160 2024-11-10T09:16:30,824 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8248691b1dc441a48fdaadafd6bac160, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T09:16:30,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7acca5a141774323a470ee4df89b65b7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7acca5a141774323a470ee4df89b65b7 2024-11-10T09:16:30,861 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7acca5a141774323a470ee4df89b65b7, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T09:16:30,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/6d889531306a48f09e496b58c5cc4555 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6d889531306a48f09e496b58c5cc4555 2024-11-10T09:16:30,879 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6d889531306a48f09e496b58c5cc4555, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T09:16:30,880 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~126.68 KB/129720, heapSize ~363.19 KB/371904, currentSize=0 B/0 for bfb4913e15c1cd99308737d49021ea5d in 267ms, sequenceid=408, compaction requested=false 2024-11-10T09:16:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:30,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T09:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T09:16:30,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T09:16:30,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2350 sec 2024-11-10T09:16:30,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2490 sec 2024-11-10T09:16:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:31,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:31,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:31,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f54a2e09d1f44816aa55b907eb23fd6b is 50, key is test_row_0/A:col10/1731230191478/Put/seqid=0 2024-11-10T09:16:31,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230191516, max timestamp=1731230191560Num uniques 859; 2024-11-10T09:16:31,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:31,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230191516, max timestamp=1731230191560Num uniques 859; 2024-11-10T09:16:31,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:31,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230191516, max timestamp=1731230191560Num uniques 859; 2024-11-10T09:16:31,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:31,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741921_1097 (size=12297) 2024-11-10T09:16:31,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f54a2e09d1f44816aa55b907eb23fd6b 2024-11-10T09:16:31,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d65e5719022d4bb8907ff69118d5e057 is 50, key is test_row_0/B:col10/1731230191478/Put/seqid=0 2024-11-10T09:16:31,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230251629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:31,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230251637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:31,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230251641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230251644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230251649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:31,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741922_1098 (size=9857) 2024-11-10T09:16:31,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d65e5719022d4bb8907ff69118d5e057 2024-11-10T09:16:31,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/1f39d48639e8410da9d2ca567954ef95 is 50, key is test_row_0/C:col10/1731230191478/Put/seqid=0 2024-11-10T09:16:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741923_1099 (size=9857) 2024-11-10T09:16:31,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/1f39d48639e8410da9d2ca567954ef95 2024-11-10T09:16:31,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f54a2e09d1f44816aa55b907eb23fd6b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f54a2e09d1f44816aa55b907eb23fd6b 2024-11-10T09:16:31,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f54a2e09d1f44816aa55b907eb23fd6b, entries=150, sequenceid=421, filesize=12.0 K 2024-11-10T09:16:31,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d65e5719022d4bb8907ff69118d5e057 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d65e5719022d4bb8907ff69118d5e057 2024-11-10T09:16:31,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d65e5719022d4bb8907ff69118d5e057, entries=100, sequenceid=421, filesize=9.6 K 2024-11-10T09:16:31,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/1f39d48639e8410da9d2ca567954ef95 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f39d48639e8410da9d2ca567954ef95 2024-11-10T09:16:31,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f39d48639e8410da9d2ca567954ef95, entries=100, sequenceid=421, filesize=9.6 K 2024-11-10T09:16:31,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bfb4913e15c1cd99308737d49021ea5d in 287ms, sequenceid=421, compaction requested=true 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:31,766 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:31,766 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:31,768 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37785 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:31,768 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:31,768 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:31,768 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/4604c155c6f94744966e49b71e7929fb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8248691b1dc441a48fdaadafd6bac160, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f54a2e09d1f44816aa55b907eb23fd6b] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=36.9 K 2024-11-10T09:16:31,769 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4604c155c6f94744966e49b71e7929fb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731230187270 2024-11-10T09:16:31,770 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8248691b1dc441a48fdaadafd6bac160, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230189262 2024-11-10T09:16:31,770 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:31,770 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:31,770 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:31,771 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/ee81ca86628f45a29dbb674a0f1a3f67, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7acca5a141774323a470ee4df89b65b7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d65e5719022d4bb8907ff69118d5e057] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=34.5 K 2024-11-10T09:16:31,771 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54a2e09d1f44816aa55b907eb23fd6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731230191474 2024-11-10T09:16:31,772 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ee81ca86628f45a29dbb674a0f1a3f67, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731230187270 2024-11-10T09:16:31,773 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7acca5a141774323a470ee4df89b65b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230189262 2024-11-10T09:16:31,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731230191601, max timestamp=1731230191771Num uniques 1018; 2024-11-10T09:16:31,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731230191601, max timestamp=1731230191771Num uniques 1018; 2024-11-10T09:16:31,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T09:16:31,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T09:16:31,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-10T09:16:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:31,779 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d65e5719022d4bb8907ff69118d5e057, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731230191474 2024-11-10T09:16:31,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731230191601, max timestamp=1731230191771Num uniques 1018; 2024-11-10T09:16:31,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T09:16:31,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:31,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:31,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:31,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:31,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:31,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:31,794 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:31,797 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/7d15823aa4f44fd6a77dc1f7d7ff4196 is 50, key is test_row_0/A:col10/1731230191478/Put/seqid=0 2024-11-10T09:16:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:16:31,807 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:31,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:31,812 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:31,812 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/31ebb444b0114a2f96b2115bf3e8b1d6 is 50, key is test_row_0/B:col10/1731230191478/Put/seqid=0 2024-11-10T09:16:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T09:16:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:16:31,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/eec3445ce84548dc95d83baa01f16a89 is 50, key is test_row_0/A:col10/1731230191771/Put/seqid=0 2024-11-10T09:16:31,814 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:31,815 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:31,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741924_1100 (size=13289) 2024-11-10T09:16:31,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741926_1102 (size=12301) 2024-11-10T09:16:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741925_1101 (size=13289) 2024-11-10T09:16:31,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/eec3445ce84548dc95d83baa01f16a89 2024-11-10T09:16:31,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230191804, max timestamp=1731230191848Num uniques 859; 2024-11-10T09:16:31,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230191804, max timestamp=1731230191848Num uniques 859; 2024-11-10T09:16:31,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:31,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:31,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230191804, max timestamp=1731230191848Num uniques 859; 2024-11-10T09:16:31,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:31,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/f147b1b3b7aa4abe993facb3bab2f9c9 is 50, key is test_row_0/B:col10/1731230191771/Put/seqid=0 2024-11-10T09:16:31,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741927_1103 (size=12301) 2024-11-10T09:16:31,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/f147b1b3b7aa4abe993facb3bab2f9c9 2024-11-10T09:16:31,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/c485c1edb98c43adb08272d4099be449 is 50, key is test_row_0/C:col10/1731230191771/Put/seqid=0 2024-11-10T09:16:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:16:31,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741928_1104 (size=12301) 2024-11-10T09:16:31,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T09:16:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:31,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230191983, max timestamp=1731230191994Num uniques 1018; 2024-11-10T09:16:32,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230191983, max timestamp=1731230191994Num uniques 1068; 2024-11-10T09:16:32,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T09:16:32,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T09:16:32,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230191983, max timestamp=1731230191994Num uniques 1068; 2024-11-10T09:16:32,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T09:16:32,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731230192027, max timestamp=1731230192069Num uniques 1077; 2024-11-10T09:16:32,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-10T09:16:32,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731230192027, max timestamp=1731230192069Num uniques 1077; 2024-11-10T09:16:32,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-10T09:16:32,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731230192027, max timestamp=1731230192069Num uniques 1077; 2024-11-10T09:16:32,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-10T09:16:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:16:32,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T09:16:32,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:32,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:32,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:32,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230252147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230252150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230252152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230252166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230252176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,230 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/7d15823aa4f44fd6a77dc1f7d7ff4196 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/7d15823aa4f44fd6a77dc1f7d7ff4196 2024-11-10T09:16:32,243 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into 7d15823aa4f44fd6a77dc1f7d7ff4196(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:32,243 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:32,243 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=13, startTime=1731230191766; duration=0sec 2024-11-10T09:16:32,245 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:32,245 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:32,245 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:32,247 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/31ebb444b0114a2f96b2115bf3e8b1d6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/31ebb444b0114a2f96b2115bf3e8b1d6 2024-11-10T09:16:32,249 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:32,249 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:32,249 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:32,249 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/a571df21cff34e36857ee248c17948f2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6d889531306a48f09e496b58c5cc4555, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f39d48639e8410da9d2ca567954ef95] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=34.5 K 2024-11-10T09:16:32,250 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a571df21cff34e36857ee248c17948f2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731230187270 2024-11-10T09:16:32,250 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d889531306a48f09e496b58c5cc4555, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230189262 2024-11-10T09:16:32,250 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f39d48639e8410da9d2ca567954ef95, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731230191474 2024-11-10T09:16:32,260 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 31ebb444b0114a2f96b2115bf3e8b1d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:32,260 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:32,260 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=13, startTime=1731230191766; duration=0sec 2024-11-10T09:16:32,260 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:32,260 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:32,264 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:32,265 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/d6aa23abd2e74b6c854d6c2b9e39a7e8 is 50, key is test_row_0/C:col10/1731230191478/Put/seqid=0 2024-11-10T09:16:32,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741929_1105 (size=13289) 2024-11-10T09:16:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230252278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230252278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,294 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/d6aa23abd2e74b6c854d6c2b9e39a7e8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/d6aa23abd2e74b6c854d6c2b9e39a7e8 2024-11-10T09:16:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230252287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T09:16:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:32,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:32,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230252297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230252309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/c485c1edb98c43adb08272d4099be449 2024-11-10T09:16:32,332 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into d6aa23abd2e74b6c854d6c2b9e39a7e8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:32,332 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:32,332 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=13, startTime=1731230191766; duration=0sec 2024-11-10T09:16:32,332 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:32,332 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:32,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/eec3445ce84548dc95d83baa01f16a89 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/eec3445ce84548dc95d83baa01f16a89 2024-11-10T09:16:32,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/eec3445ce84548dc95d83baa01f16a89, entries=150, sequenceid=460, filesize=12.0 K 2024-11-10T09:16:32,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/f147b1b3b7aa4abe993facb3bab2f9c9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/f147b1b3b7aa4abe993facb3bab2f9c9 2024-11-10T09:16:32,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/f147b1b3b7aa4abe993facb3bab2f9c9, entries=150, sequenceid=460, filesize=12.0 K 2024-11-10T09:16:32,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/c485c1edb98c43adb08272d4099be449 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/c485c1edb98c43adb08272d4099be449 2024-11-10T09:16:32,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/c485c1edb98c43adb08272d4099be449, entries=150, sequenceid=460, filesize=12.0 K 2024-11-10T09:16:32,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for bfb4913e15c1cd99308737d49021ea5d in 587ms, sequenceid=460, compaction requested=false 2024-11-10T09:16:32,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:16:32,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T09:16:32,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:32,467 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=154.31 KB heapSize=429 KB 2024-11-10T09:16:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/41add591f654402b94ed79c272782b3b is 50, key is test_row_0/A:col10/1731230192141/Put/seqid=0 2024-11-10T09:16:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:32,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741930_1106 (size=12301) 2024-11-10T09:16:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230252520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230252548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230252549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230252551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230252552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230252668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230252668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230252668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230252668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230252858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230252888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230252888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230252888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230252888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:32,926 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/41add591f654402b94ed79c272782b3b 2024-11-10T09:16:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/153b14c8718c4915bf1ccaebd02a2fe7 is 50, key is test_row_0/B:col10/1731230192141/Put/seqid=0 2024-11-10T09:16:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:16:32,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741931_1107 (size=12301) 2024-11-10T09:16:32,988 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/153b14c8718c4915bf1ccaebd02a2fe7 2024-11-10T09:16:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/081fdeac543f47029357745da3c6709c is 50, key is test_row_0/C:col10/1731230192141/Put/seqid=0 2024-11-10T09:16:33,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741932_1108 (size=12301) 2024-11-10T09:16:33,026 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/081fdeac543f47029357745da3c6709c 2024-11-10T09:16:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/41add591f654402b94ed79c272782b3b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/41add591f654402b94ed79c272782b3b 2024-11-10T09:16:33,047 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/41add591f654402b94ed79c272782b3b, entries=150, sequenceid=534, filesize=12.0 K 2024-11-10T09:16:33,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T09:16:33,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/153b14c8718c4915bf1ccaebd02a2fe7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/153b14c8718c4915bf1ccaebd02a2fe7 2024-11-10T09:16:33,056 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/153b14c8718c4915bf1ccaebd02a2fe7, entries=150, sequenceid=534, filesize=12.0 K 2024-11-10T09:16:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/081fdeac543f47029357745da3c6709c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/081fdeac543f47029357745da3c6709c 2024-11-10T09:16:33,070 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/081fdeac543f47029357745da3c6709c, entries=150, sequenceid=534, filesize=12.0 K 2024-11-10T09:16:33,072 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~428.95 KB/439248, currentSize=33.54 KB/34350 for bfb4913e15c1cd99308737d49021ea5d in 605ms, sequenceid=534, compaction requested=true 2024-11-10T09:16:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T09:16:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T09:16:33,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T09:16:33,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-11-10T09:16:33,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2750 sec 2024-11-10T09:16:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:33,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:33,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/33c701a1e87f48779fdad074838885b5 is 50, key is test_row_0/A:col10/1731230193212/Put/seqid=0 2024-11-10T09:16:33,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741933_1109 (size=12301) 2024-11-10T09:16:33,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/33c701a1e87f48779fdad074838885b5 2024-11-10T09:16:33,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731230193253, max timestamp=1731230193280Num uniques 859; 2024-11-10T09:16:33,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T09:16:33,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731230193253, max timestamp=1731230193280Num uniques 859; 2024-11-10T09:16:33,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731230193253, max timestamp=1731230193280Num uniques 859; 2024-11-10T09:16:33,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T09:16:33,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T09:16:33,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d3d20453a56f4a6eb1ba6a5c2c87b039 is 50, key is test_row_0/B:col10/1731230193212/Put/seqid=0 2024-11-10T09:16:33,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741934_1110 (size=12301) 2024-11-10T09:16:33,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230253332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d3d20453a56f4a6eb1ba6a5c2c87b039 2024-11-10T09:16:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230253335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230253341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230253338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/2f1812c7620f45069a193210d7fcb2cf is 50, key is test_row_0/C:col10/1731230193212/Put/seqid=0 2024-11-10T09:16:33,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741935_1111 (size=12301) 2024-11-10T09:16:33,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/2f1812c7620f45069a193210d7fcb2cf 2024-11-10T09:16:33,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230253368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/33c701a1e87f48779fdad074838885b5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/33c701a1e87f48779fdad074838885b5 2024-11-10T09:16:33,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/33c701a1e87f48779fdad074838885b5, entries=150, sequenceid=545, filesize=12.0 K 2024-11-10T09:16:33,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d3d20453a56f4a6eb1ba6a5c2c87b039 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d3d20453a56f4a6eb1ba6a5c2c87b039 2024-11-10T09:16:33,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d3d20453a56f4a6eb1ba6a5c2c87b039, entries=150, sequenceid=545, filesize=12.0 K 2024-11-10T09:16:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/2f1812c7620f45069a193210d7fcb2cf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/2f1812c7620f45069a193210d7fcb2cf 2024-11-10T09:16:33,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/2f1812c7620f45069a193210d7fcb2cf, entries=150, sequenceid=545, filesize=12.0 K 2024-11-10T09:16:33,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for bfb4913e15c1cd99308737d49021ea5d in 208ms, sequenceid=545, compaction requested=true 2024-11-10T09:16:33,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:33,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:33,422 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:33,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:33,422 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:33,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:33,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:33,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:33,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:33,423 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:33,423 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/A is initiating minor compaction (all files) 2024-11-10T09:16:33,424 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/A in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:33,424 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/7d15823aa4f44fd6a77dc1f7d7ff4196, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/eec3445ce84548dc95d83baa01f16a89, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/41add591f654402b94ed79c272782b3b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/33c701a1e87f48779fdad074838885b5] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=49.0 K 2024-11-10T09:16:33,424 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d15823aa4f44fd6a77dc1f7d7ff4196, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731230189262 2024-11-10T09:16:33,425 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:33,425 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/B is initiating minor compaction (all files) 2024-11-10T09:16:33,425 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec3445ce84548dc95d83baa01f16a89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731230191645 2024-11-10T09:16:33,425 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/B in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:33,425 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/31ebb444b0114a2f96b2115bf3e8b1d6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/f147b1b3b7aa4abe993facb3bab2f9c9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/153b14c8718c4915bf1ccaebd02a2fe7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d3d20453a56f4a6eb1ba6a5c2c87b039] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=49.0 K 2024-11-10T09:16:33,425 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41add591f654402b94ed79c272782b3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731230192139 2024-11-10T09:16:33,426 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 31ebb444b0114a2f96b2115bf3e8b1d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731230189262 2024-11-10T09:16:33,426 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c701a1e87f48779fdad074838885b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1731230192500 2024-11-10T09:16:33,426 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f147b1b3b7aa4abe993facb3bab2f9c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731230191645 2024-11-10T09:16:33,427 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 153b14c8718c4915bf1ccaebd02a2fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731230192139 2024-11-10T09:16:33,427 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d20453a56f4a6eb1ba6a5c2c87b039, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1731230192500 2024-11-10T09:16:33,455 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:33,456 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7bf58eb853784eca853854774e6e4f30 is 50, key is test_row_0/B:col10/1731230193212/Put/seqid=0 2024-11-10T09:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:33,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-11-10T09:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:33,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:33,473 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#A#compaction#100 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:33,474 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/beeaa2e2f15e48dd8d6c840fc5532ba9 is 50, key is test_row_0/A:col10/1731230193212/Put/seqid=0 2024-11-10T09:16:33,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f573f7c9500b4376aecaf63de8e9a1f3 is 50, key is test_row_0/A:col10/1731230193336/Put/seqid=0 2024-11-10T09:16:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230253489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230253492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230253497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230253499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741936_1112 (size=13425) 2024-11-10T09:16:33,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741937_1113 (size=13425) 2024-11-10T09:16:33,563 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/beeaa2e2f15e48dd8d6c840fc5532ba9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/beeaa2e2f15e48dd8d6c840fc5532ba9 2024-11-10T09:16:33,572 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/A of bfb4913e15c1cd99308737d49021ea5d into beeaa2e2f15e48dd8d6c840fc5532ba9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:33,572 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:33,572 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/A, priority=12, startTime=1731230193421; duration=0sec 2024-11-10T09:16:33,572 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:33,573 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:33,573 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:33,578 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:33,579 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): bfb4913e15c1cd99308737d49021ea5d/C is initiating minor compaction (all files) 2024-11-10T09:16:33,579 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfb4913e15c1cd99308737d49021ea5d/C in TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:33,579 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/d6aa23abd2e74b6c854d6c2b9e39a7e8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/c485c1edb98c43adb08272d4099be449, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/081fdeac543f47029357745da3c6709c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/2f1812c7620f45069a193210d7fcb2cf] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp, totalSize=49.0 K 2024-11-10T09:16:33,580 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6aa23abd2e74b6c854d6c2b9e39a7e8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731230189262 2024-11-10T09:16:33,580 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c485c1edb98c43adb08272d4099be449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731230191645 2024-11-10T09:16:33,580 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081fdeac543f47029357745da3c6709c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731230192139 2024-11-10T09:16:33,581 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f1812c7620f45069a193210d7fcb2cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1731230192500 2024-11-10T09:16:33,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741938_1114 (size=12301) 2024-11-10T09:16:33,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f573f7c9500b4376aecaf63de8e9a1f3 2024-11-10T09:16:33,611 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfb4913e15c1cd99308737d49021ea5d#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:33,611 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/25446cb7e7ab4c43b4215ef62eae01ad is 50, key is test_row_0/C:col10/1731230193212/Put/seqid=0 2024-11-10T09:16:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230253607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230253608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230253608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/6daf55c1a9004a96822862dcb3672c23 is 50, key is test_row_0/B:col10/1731230193336/Put/seqid=0 2024-11-10T09:16:33,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230253617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741939_1115 (size=13425) 2024-11-10T09:16:33,650 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/25446cb7e7ab4c43b4215ef62eae01ad as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/25446cb7e7ab4c43b4215ef62eae01ad 2024-11-10T09:16:33,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741940_1116 (size=12301) 2024-11-10T09:16:33,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/6daf55c1a9004a96822862dcb3672c23 2024-11-10T09:16:33,667 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/C of bfb4913e15c1cd99308737d49021ea5d into 25446cb7e7ab4c43b4215ef62eae01ad(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:33,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:33,667 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/C, priority=12, startTime=1731230193422; duration=0sec 2024-11-10T09:16:33,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:33,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:33,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/04339bc28a59432e9c71672733c260c9 is 50, key is test_row_0/C:col10/1731230193336/Put/seqid=0 2024-11-10T09:16:33,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741941_1117 (size=12301) 2024-11-10T09:16:33,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230253817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230253827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230253827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230253827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:33,935 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/7bf58eb853784eca853854774e6e4f30 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7bf58eb853784eca853854774e6e4f30 2024-11-10T09:16:33,942 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfb4913e15c1cd99308737d49021ea5d/B of bfb4913e15c1cd99308737d49021ea5d into 7bf58eb853784eca853854774e6e4f30(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:33,942 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:33,942 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., storeName=bfb4913e15c1cd99308737d49021ea5d/B, priority=12, startTime=1731230193422; duration=0sec 2024-11-10T09:16:33,942 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:33,942 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:16:33,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:33,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-10T09:16:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:16:34,000 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:34,001 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:34,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:16:34,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/04339bc28a59432e9c71672733c260c9 2024-11-10T09:16:34,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230254127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f573f7c9500b4376aecaf63de8e9a1f3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f573f7c9500b4376aecaf63de8e9a1f3 2024-11-10T09:16:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230254138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230254139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f573f7c9500b4376aecaf63de8e9a1f3, entries=150, sequenceid=580, filesize=12.0 K 2024-11-10T09:16:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230254139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/6daf55c1a9004a96822862dcb3672c23 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6daf55c1a9004a96822862dcb3672c23 2024-11-10T09:16:34,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:16:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:34,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:34,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:34,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:34,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:34,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6daf55c1a9004a96822862dcb3672c23, entries=150, sequenceid=580, filesize=12.0 K 2024-11-10T09:16:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/04339bc28a59432e9c71672733c260c9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/04339bc28a59432e9c71672733c260c9 2024-11-10T09:16:34,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/04339bc28a59432e9c71672733c260c9, entries=150, sequenceid=580, filesize=12.0 K 2024-11-10T09:16:34,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~421.27 KB/431376, currentSize=33.54 KB/34350 for bfb4913e15c1cd99308737d49021ea5d in 706ms, sequenceid=580, compaction requested=false 2024-11-10T09:16:34,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:34,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:16:34,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:34,315 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T09:16:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:16:34,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/bdec02c93c574ded80ed9826bebffa71 is 50, key is test_row_0/A:col10/1731230193483/Put/seqid=0 2024-11-10T09:16:34,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741942_1118 (size=9857) 2024-11-10T09:16:34,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. as already flushing 2024-11-10T09:16:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:34,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230194433, max timestamp=1731230194475Num uniques 859; 2024-11-10T09:16:34,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230194433, max timestamp=1731230194475Num uniques 859; 2024-11-10T09:16:34,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:16:34,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:16:34,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230194433, max timestamp=1731230194475Num uniques 859; 2024-11-10T09:16:34,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:16:34,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230254618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:16:34,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55920 deadline: 1731230254637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55896 deadline: 1731230254658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55912 deadline: 1731230254663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55882 deadline: 1731230254663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,727 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T09:16:34,727 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:34,735 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,737 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/bdec02c93c574ded80ed9826bebffa71 2024-11-10T09:16:34,739 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:34,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230254733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:34,747 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T09:16:34,747 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:34,747 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,748 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T09:16:34,748 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:34,748 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,748 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:34,748 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:34,754 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T09:16:34,754 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:34,754 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,754 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:34,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:34,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1cfc67767e594c8293f806761a219dfa is 50, key is test_row_0/B:col10/1731230193483/Put/seqid=0 2024-11-10T09:16:34,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741943_1119 (size=9857) 2024-11-10T09:16:34,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230254957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:16:35,164 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1cfc67767e594c8293f806761a219dfa 2024-11-10T09:16:35,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/35a28d6468424f98ba7ccbfd866a1b6a is 50, key is test_row_0/C:col10/1731230193483/Put/seqid=0 2024-11-10T09:16:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741944_1120 (size=9857) 2024-11-10T09:16:35,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55898 deadline: 1731230255267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:16:35,579 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/35a28d6468424f98ba7ccbfd866a1b6a 2024-11-10T09:16:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/bdec02c93c574ded80ed9826bebffa71 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bdec02c93c574ded80ed9826bebffa71 2024-11-10T09:16:35,593 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bdec02c93c574ded80ed9826bebffa71, entries=100, sequenceid=591, filesize=9.6 K 2024-11-10T09:16:35,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/1cfc67767e594c8293f806761a219dfa as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1cfc67767e594c8293f806761a219dfa 2024-11-10T09:16:35,599 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1cfc67767e594c8293f806761a219dfa, entries=100, sequenceid=591, filesize=9.6 K 2024-11-10T09:16:35,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/35a28d6468424f98ba7ccbfd866a1b6a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/35a28d6468424f98ba7ccbfd866a1b6a 2024-11-10T09:16:35,605 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/35a28d6468424f98ba7ccbfd866a1b6a, entries=100, sequenceid=591, filesize=9.6 K 2024-11-10T09:16:35,606 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for bfb4913e15c1cd99308737d49021ea5d in 1291ms, sequenceid=591, compaction requested=true 2024-11-10T09:16:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-10T09:16:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-10T09:16:35,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T09:16:35,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6060 sec 2024-11-10T09:16:35,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6380 sec 2024-11-10T09:16:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:35,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-10T09:16:35,659 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T09:16:35,659 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:35,659 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:16:35,659 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:35,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:16:35,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/386504bec3db4a2f945b693f6c609b98 is 50, key is test_row_0/A:col10/1731230194586/Put/seqid=0 2024-11-10T09:16:35,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741945_1121 (size=12301) 2024-11-10T09:16:35,679 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T09:16:35,679 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:35,679 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T09:16:35,679 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,679 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:35,679 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,679 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,679 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T09:16:35,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:35,679 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:35,679 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,679 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,679 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:35,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:35,779 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T09:16:35,780 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:16:35,780 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,780 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:35,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:36,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/386504bec3db4a2f945b693f6c609b98 2024-11-10T09:16:36,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/b0312eaa42594a3cacf2d0a33a492883 is 50, key is test_row_0/B:col10/1731230194586/Put/seqid=0 2024-11-10T09:16:36,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741946_1122 (size=12301) 2024-11-10T09:16:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:16:36,147 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:36,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1826 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1804 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 884 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2652 rows 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 892 2024-11-10T09:16:36,148 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2676 rows 2024-11-10T09:16:36,148 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:16:36,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:16:36,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:36,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:36,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:16:36,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:36,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230196158"}]},"ts":"1731230196158"} 2024-11-10T09:16:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:36,160 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:16:36,161 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:16:36,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:16:36,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, UNASSIGN}] 2024-11-10T09:16:36,167 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, UNASSIGN 2024-11-10T09:16:36,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=bfb4913e15c1cd99308737d49021ea5d, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:36,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, UNASSIGN because future has completed 2024-11-10T09:16:36,170 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:16:36,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:16:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:36,328 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:36,328 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:16:36,329 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing bfb4913e15c1cd99308737d49021ea5d, disabling compactions & flushes 2024-11-10T09:16:36,329 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:36,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/b0312eaa42594a3cacf2d0a33a492883 2024-11-10T09:16:36,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/59bf7e34b841424fa779c9360371b6b5 is 50, key is test_row_0/C:col10/1731230194586/Put/seqid=0 2024-11-10T09:16:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741947_1123 (size=12301) 2024-11-10T09:16:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:36,804 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:16:36,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/59bf7e34b841424fa779c9360371b6b5 2024-11-10T09:16:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/386504bec3db4a2f945b693f6c609b98 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/386504bec3db4a2f945b693f6c609b98 2024-11-10T09:16:36,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/386504bec3db4a2f945b693f6c609b98, entries=150, sequenceid=629, filesize=12.0 K 2024-11-10T09:16:36,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/b0312eaa42594a3cacf2d0a33a492883 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/b0312eaa42594a3cacf2d0a33a492883 2024-11-10T09:16:36,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/b0312eaa42594a3cacf2d0a33a492883, entries=150, sequenceid=629, filesize=12.0 K 2024-11-10T09:16:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/59bf7e34b841424fa779c9360371b6b5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/59bf7e34b841424fa779c9360371b6b5 2024-11-10T09:16:36,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/59bf7e34b841424fa779c9360371b6b5, entries=150, sequenceid=629, filesize=12.0 K 2024-11-10T09:16:36,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for bfb4913e15c1cd99308737d49021ea5d in 1278ms, sequenceid=629, compaction requested=true 2024-11-10T09:16:36,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfb4913e15c1cd99308737d49021ea5d: 2024-11-10T09:16:36,937 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:36,937 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:36,937 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. after waiting 0 ms 2024-11-10T09:16:36,937 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:36,937 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. because compaction request was cancelled 2024-11-10T09:16:36,937 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:A 2024-11-10T09:16:36,937 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing bfb4913e15c1cd99308737d49021ea5d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T09:16:36,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:A, priority=-2147483648, current under compaction store size is 0 2024-11-10T09:16:36,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:36,938 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. because compaction request was cancelled 2024-11-10T09:16:36,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:B, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:36,938 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:B 2024-11-10T09:16:36,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:36,938 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. because compaction request was cancelled 2024-11-10T09:16:36,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=A 2024-11-10T09:16:36,938 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfb4913e15c1cd99308737d49021ea5d:C 2024-11-10T09:16:36,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfb4913e15c1cd99308737d49021ea5d:C, priority=-2147483648, current under compaction store size is 0 2024-11-10T09:16:36,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:36,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:36,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=B 2024-11-10T09:16:36,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:36,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bfb4913e15c1cd99308737d49021ea5d, store=C 2024-11-10T09:16:36,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:36,946 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f58fbc03da4e410e8b080430b70dcdac is 50, key is test_row_0/A:col10/1731230195678/Put/seqid=0 2024-11-10T09:16:36,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741948_1124 (size=9857) 2024-11-10T09:16:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:37,366 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f58fbc03da4e410e8b080430b70dcdac 2024-11-10T09:16:37,378 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d2aea5dfc47e4efabeca25aef5af8113 is 50, key is test_row_0/B:col10/1731230195678/Put/seqid=0 2024-11-10T09:16:37,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741949_1125 (size=9857) 2024-11-10T09:16:37,791 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d2aea5dfc47e4efabeca25aef5af8113 2024-11-10T09:16:37,798 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/500791b5681b45999021ee6db47090b0 is 50, key is test_row_0/C:col10/1731230195678/Put/seqid=0 2024-11-10T09:16:37,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741950_1126 (size=9857) 2024-11-10T09:16:38,203 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/500791b5681b45999021ee6db47090b0 2024-11-10T09:16:38,208 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/A/f58fbc03da4e410e8b080430b70dcdac as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f58fbc03da4e410e8b080430b70dcdac 2024-11-10T09:16:38,213 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f58fbc03da4e410e8b080430b70dcdac, entries=100, sequenceid=636, filesize=9.6 K 2024-11-10T09:16:38,214 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/B/d2aea5dfc47e4efabeca25aef5af8113 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d2aea5dfc47e4efabeca25aef5af8113 2024-11-10T09:16:38,219 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d2aea5dfc47e4efabeca25aef5af8113, entries=100, sequenceid=636, filesize=9.6 K 2024-11-10T09:16:38,220 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/.tmp/C/500791b5681b45999021ee6db47090b0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/500791b5681b45999021ee6db47090b0 2024-11-10T09:16:38,225 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/500791b5681b45999021ee6db47090b0, entries=100, sequenceid=636, filesize=9.6 K 2024-11-10T09:16:38,226 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for bfb4913e15c1cd99308737d49021ea5d in 1289ms, sequenceid=636, compaction requested=true 2024-11-10T09:16:38,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/60a79dfd262e4998bdabd23eb0ad30d4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dd7166a1764e41b89cbe277a85633925, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/2b01e31db0254331ad3c071b500ad4a2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/596ccf60ac954237a4c1b422a77c6824, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bfb554bb12514c88a255cd1b1933b396, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5d44ed13b7b5487a8f8b2958b9bf0ba3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/1f708bde6b774a3fb0c2cb3840721f23, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/af70c510fbab450dac7b2843d31ba673, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5c68c2cdcfcb4f34a77e8f9db12bbc31, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/cc27a747617a40c6938deb63a3ef8e17, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/9d817c83d8804d2983d7417ce16fa436, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98d7c0a6363146a591355524f218c397, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/52228b521c924ba2aa7ecf6d516200f7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8cea077cf9d84868ba890e4b416d7bb9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/ce49e3c72be748d2aea3e34d6ec2f551, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/da0fe346617948c2ab81583a8394a9b2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8a36d31fbc654d35aaded27804647347, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/0dfcfa349f98458b8900038fde8707d9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/523f85bf8f01494e923bc625ddc67ff4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/d3aa287b7b224f5a8fdffebc959ca106, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/e3c8508c05c9466095875de8eb246695, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/34411a0630354a8dbe722dd1dc10bfc9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98589f40cdc645129141c23623839534, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/785613e214b341f586f131e77c86ad8a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/4604c155c6f94744966e49b71e7929fb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/3e82c1c911984f02a3d1e51064a9be93, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8248691b1dc441a48fdaadafd6bac160, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/7d15823aa4f44fd6a77dc1f7d7ff4196, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f54a2e09d1f44816aa55b907eb23fd6b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/eec3445ce84548dc95d83baa01f16a89, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/41add591f654402b94ed79c272782b3b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/33c701a1e87f48779fdad074838885b5] to archive 2024-11-10T09:16:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:16:38,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/60a79dfd262e4998bdabd23eb0ad30d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/60a79dfd262e4998bdabd23eb0ad30d4 2024-11-10T09:16:38,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dd7166a1764e41b89cbe277a85633925 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dd7166a1764e41b89cbe277a85633925 2024-11-10T09:16:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/2b01e31db0254331ad3c071b500ad4a2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/2b01e31db0254331ad3c071b500ad4a2 2024-11-10T09:16:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/596ccf60ac954237a4c1b422a77c6824 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/596ccf60ac954237a4c1b422a77c6824 2024-11-10T09:16:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bfb554bb12514c88a255cd1b1933b396 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bfb554bb12514c88a255cd1b1933b396 2024-11-10T09:16:38,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5d44ed13b7b5487a8f8b2958b9bf0ba3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5d44ed13b7b5487a8f8b2958b9bf0ba3 2024-11-10T09:16:38,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/1f708bde6b774a3fb0c2cb3840721f23 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/1f708bde6b774a3fb0c2cb3840721f23 2024-11-10T09:16:38,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/af70c510fbab450dac7b2843d31ba673 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/af70c510fbab450dac7b2843d31ba673 2024-11-10T09:16:38,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5c68c2cdcfcb4f34a77e8f9db12bbc31 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/5c68c2cdcfcb4f34a77e8f9db12bbc31 2024-11-10T09:16:38,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/dfee37e2f9aa4fdbaa7b3a277ddaf8b1 2024-11-10T09:16:38,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/cc27a747617a40c6938deb63a3ef8e17 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/cc27a747617a40c6938deb63a3ef8e17 2024-11-10T09:16:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/9d817c83d8804d2983d7417ce16fa436 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/9d817c83d8804d2983d7417ce16fa436 2024-11-10T09:16:38,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98d7c0a6363146a591355524f218c397 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98d7c0a6363146a591355524f218c397 2024-11-10T09:16:38,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/52228b521c924ba2aa7ecf6d516200f7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/52228b521c924ba2aa7ecf6d516200f7 2024-11-10T09:16:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8cea077cf9d84868ba890e4b416d7bb9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8cea077cf9d84868ba890e4b416d7bb9 2024-11-10T09:16:38,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/ce49e3c72be748d2aea3e34d6ec2f551 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/ce49e3c72be748d2aea3e34d6ec2f551 2024-11-10T09:16:38,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/da0fe346617948c2ab81583a8394a9b2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/da0fe346617948c2ab81583a8394a9b2 2024-11-10T09:16:38,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8a36d31fbc654d35aaded27804647347 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8a36d31fbc654d35aaded27804647347 2024-11-10T09:16:38,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/0dfcfa349f98458b8900038fde8707d9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/0dfcfa349f98458b8900038fde8707d9 2024-11-10T09:16:38,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/523f85bf8f01494e923bc625ddc67ff4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/523f85bf8f01494e923bc625ddc67ff4 2024-11-10T09:16:38,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/d3aa287b7b224f5a8fdffebc959ca106 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/d3aa287b7b224f5a8fdffebc959ca106 2024-11-10T09:16:38,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/e3c8508c05c9466095875de8eb246695 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/e3c8508c05c9466095875de8eb246695 2024-11-10T09:16:38,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/34411a0630354a8dbe722dd1dc10bfc9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/34411a0630354a8dbe722dd1dc10bfc9 2024-11-10T09:16:38,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98589f40cdc645129141c23623839534 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/98589f40cdc645129141c23623839534 2024-11-10T09:16:38,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/785613e214b341f586f131e77c86ad8a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/785613e214b341f586f131e77c86ad8a 2024-11-10T09:16:38,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/4604c155c6f94744966e49b71e7929fb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/4604c155c6f94744966e49b71e7929fb 2024-11-10T09:16:38,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/3e82c1c911984f02a3d1e51064a9be93 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/3e82c1c911984f02a3d1e51064a9be93 2024-11-10T09:16:38,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8248691b1dc441a48fdaadafd6bac160 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/8248691b1dc441a48fdaadafd6bac160 2024-11-10T09:16:38,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/7d15823aa4f44fd6a77dc1f7d7ff4196 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/7d15823aa4f44fd6a77dc1f7d7ff4196 2024-11-10T09:16:38,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f54a2e09d1f44816aa55b907eb23fd6b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f54a2e09d1f44816aa55b907eb23fd6b 2024-11-10T09:16:38,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/eec3445ce84548dc95d83baa01f16a89 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/eec3445ce84548dc95d83baa01f16a89 2024-11-10T09:16:38,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/41add591f654402b94ed79c272782b3b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/41add591f654402b94ed79c272782b3b 2024-11-10T09:16:38,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/33c701a1e87f48779fdad074838885b5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/33c701a1e87f48779fdad074838885b5 2024-11-10T09:16:38,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca3267b70104dc7aa98b4f62bf26c63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/98566a4f78d349729f9b16b706e3fb68, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0d591b6007654381a9bc1151d3c44517, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/e7c15d2f19f74f87a48a0fe830966145, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/bb5cda50b9c54a739c53a055d575b691, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/67b13496de7241c39059b22b1f091fe5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0c51f8cbfcc541b7b91b232366cfd3ce, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/12e7e50550e94de4abf8e13007d255c1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e8f58a2af6c44f59440398ff774e308, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c3da2b9c05fd4872b5edb0dbf78546bf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/4791314e30684fe4b9bd35d7ad9a61bb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca1cd6a9987426b9c7ca49c1700bf90, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7800377cb4a44392bca20761954ae385, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5f8a77d66efa4624a235ef6656910b1f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5c07406c942e43e29691d09f81be5123, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/9b95d891e24349afb932c23e9d4ee466, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/2372d2f715034d4cbda61057acee8c62, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c17662004f274219acaa7a857cf97000, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/af495715bc824daabe00b3537b1f7ccd, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/771a5b1a72b5493a9638c95a674640a3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/991b07213cf343ca9fc9095d9170eec8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6ab3862015e54debb5669f1ae25e6e40, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/edb4aabb7568477cbcaa48c717b010ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/29a7dcce2f8a4d91a31df791fdd93562, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1c2e9416d447435d8563540c6fe0f87e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/ee81ca86628f45a29dbb674a0f1a3f67, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e0259eeea734b2c96526645ff81bc9d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7acca5a141774323a470ee4df89b65b7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/31ebb444b0114a2f96b2115bf3e8b1d6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d65e5719022d4bb8907ff69118d5e057, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/f147b1b3b7aa4abe993facb3bab2f9c9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/153b14c8718c4915bf1ccaebd02a2fe7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d3d20453a56f4a6eb1ba6a5c2c87b039] to archive 2024-11-10T09:16:38,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:16:38,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca3267b70104dc7aa98b4f62bf26c63 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca3267b70104dc7aa98b4f62bf26c63 2024-11-10T09:16:38,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/98566a4f78d349729f9b16b706e3fb68 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/98566a4f78d349729f9b16b706e3fb68 2024-11-10T09:16:38,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0d591b6007654381a9bc1151d3c44517 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0d591b6007654381a9bc1151d3c44517 2024-11-10T09:16:38,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/e7c15d2f19f74f87a48a0fe830966145 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/e7c15d2f19f74f87a48a0fe830966145 2024-11-10T09:16:38,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/bb5cda50b9c54a739c53a055d575b691 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/bb5cda50b9c54a739c53a055d575b691 2024-11-10T09:16:38,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/67b13496de7241c39059b22b1f091fe5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/67b13496de7241c39059b22b1f091fe5 2024-11-10T09:16:38,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0c51f8cbfcc541b7b91b232366cfd3ce to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/0c51f8cbfcc541b7b91b232366cfd3ce 2024-11-10T09:16:38,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/12e7e50550e94de4abf8e13007d255c1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/12e7e50550e94de4abf8e13007d255c1 2024-11-10T09:16:38,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e8f58a2af6c44f59440398ff774e308 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e8f58a2af6c44f59440398ff774e308 2024-11-10T09:16:38,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c3da2b9c05fd4872b5edb0dbf78546bf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c3da2b9c05fd4872b5edb0dbf78546bf 2024-11-10T09:16:38,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/4791314e30684fe4b9bd35d7ad9a61bb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/4791314e30684fe4b9bd35d7ad9a61bb 2024-11-10T09:16:38,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca1cd6a9987426b9c7ca49c1700bf90 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/aca1cd6a9987426b9c7ca49c1700bf90 2024-11-10T09:16:38,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7800377cb4a44392bca20761954ae385 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7800377cb4a44392bca20761954ae385 2024-11-10T09:16:38,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5f8a77d66efa4624a235ef6656910b1f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5f8a77d66efa4624a235ef6656910b1f 2024-11-10T09:16:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:38,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5c07406c942e43e29691d09f81be5123 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/5c07406c942e43e29691d09f81be5123 2024-11-10T09:16:38,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/9b95d891e24349afb932c23e9d4ee466 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/9b95d891e24349afb932c23e9d4ee466 2024-11-10T09:16:38,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/2372d2f715034d4cbda61057acee8c62 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/2372d2f715034d4cbda61057acee8c62 2024-11-10T09:16:38,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c17662004f274219acaa7a857cf97000 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/c17662004f274219acaa7a857cf97000 2024-11-10T09:16:38,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/af495715bc824daabe00b3537b1f7ccd to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/af495715bc824daabe00b3537b1f7ccd 2024-11-10T09:16:38,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/771a5b1a72b5493a9638c95a674640a3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/771a5b1a72b5493a9638c95a674640a3 2024-11-10T09:16:38,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/991b07213cf343ca9fc9095d9170eec8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/991b07213cf343ca9fc9095d9170eec8 2024-11-10T09:16:38,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6ab3862015e54debb5669f1ae25e6e40 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6ab3862015e54debb5669f1ae25e6e40 2024-11-10T09:16:38,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/edb4aabb7568477cbcaa48c717b010ba to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/edb4aabb7568477cbcaa48c717b010ba 2024-11-10T09:16:38,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/29a7dcce2f8a4d91a31df791fdd93562 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/29a7dcce2f8a4d91a31df791fdd93562 2024-11-10T09:16:38,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1c2e9416d447435d8563540c6fe0f87e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1c2e9416d447435d8563540c6fe0f87e 2024-11-10T09:16:38,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/ee81ca86628f45a29dbb674a0f1a3f67 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/ee81ca86628f45a29dbb674a0f1a3f67 2024-11-10T09:16:38,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e0259eeea734b2c96526645ff81bc9d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1e0259eeea734b2c96526645ff81bc9d 2024-11-10T09:16:38,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7acca5a141774323a470ee4df89b65b7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7acca5a141774323a470ee4df89b65b7 2024-11-10T09:16:38,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/31ebb444b0114a2f96b2115bf3e8b1d6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/31ebb444b0114a2f96b2115bf3e8b1d6 2024-11-10T09:16:38,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d65e5719022d4bb8907ff69118d5e057 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d65e5719022d4bb8907ff69118d5e057 2024-11-10T09:16:38,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/f147b1b3b7aa4abe993facb3bab2f9c9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/f147b1b3b7aa4abe993facb3bab2f9c9 2024-11-10T09:16:38,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/153b14c8718c4915bf1ccaebd02a2fe7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/153b14c8718c4915bf1ccaebd02a2fe7 2024-11-10T09:16:38,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d3d20453a56f4a6eb1ba6a5c2c87b039 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d3d20453a56f4a6eb1ba6a5c2c87b039 2024-11-10T09:16:38,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/15c25d28d20c4d9bad127e13d4680302, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e2a9563484d14610a68670fabeeedaa7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/585639ad15ac443f8eb7b2bd34c003d1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/8d47f4b5920f43c394251bc434b48007, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/4bc38bde595947c2bdcf5c65b459483e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5ef4d2b16ee94ed0b5727f1c979f763b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fe2b8754fb6b40ffb3647bbcda1b82e4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/88bbb6af63004b09a849392d871932fe, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/38e6947bacf1455199532c7d0a93acfa, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e878c9edc1f044c8b42d2be70fccfa78, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bf1d0960d74a423bb8410c422ac604e1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/85df83e024e34ae39053b863b9170be8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/3374a8ac22004991872c04a4aeeceac0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/b2f1c6ffd5ee4da197d7ef44000c161a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6c35adf79882445ca00a07d9cb11bf7f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/7d9b8606e6514296808457c6b5957f70, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/90e96d7a36dc4bbf8153833f2243b1ca, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bce74014ea76462e8fc98ed43072e0e9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/29ee56619ced4a59b4ade67ccec6ca69, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/0eb05834e7aa4c4587a9f9fb1f5b20a9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/43646280b52a45468938001859c786d6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fcb843e46fb2434897678da7a8aeabee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5df41a5cb52f4cce8353895811236cdc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/42b4b13592d34d90b58adbec2db02f46, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/289a56a83cc84676a21373ad02f47914, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/a571df21cff34e36857ee248c17948f2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f9712b59d46493391afccdfbc2d6682, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6d889531306a48f09e496b58c5cc4555, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/d6aa23abd2e74b6c854d6c2b9e39a7e8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f39d48639e8410da9d2ca567954ef95, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/c485c1edb98c43adb08272d4099be449, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/081fdeac543f47029357745da3c6709c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/2f1812c7620f45069a193210d7fcb2cf] to archive 2024-11-10T09:16:38,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:16:38,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/15c25d28d20c4d9bad127e13d4680302 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/15c25d28d20c4d9bad127e13d4680302 2024-11-10T09:16:38,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e2a9563484d14610a68670fabeeedaa7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e2a9563484d14610a68670fabeeedaa7 2024-11-10T09:16:38,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/585639ad15ac443f8eb7b2bd34c003d1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/585639ad15ac443f8eb7b2bd34c003d1 2024-11-10T09:16:38,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/8d47f4b5920f43c394251bc434b48007 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/8d47f4b5920f43c394251bc434b48007 2024-11-10T09:16:38,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/4bc38bde595947c2bdcf5c65b459483e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/4bc38bde595947c2bdcf5c65b459483e 2024-11-10T09:16:38,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5ef4d2b16ee94ed0b5727f1c979f763b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5ef4d2b16ee94ed0b5727f1c979f763b 2024-11-10T09:16:38,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fe2b8754fb6b40ffb3647bbcda1b82e4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fe2b8754fb6b40ffb3647bbcda1b82e4 2024-11-10T09:16:38,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/88bbb6af63004b09a849392d871932fe to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/88bbb6af63004b09a849392d871932fe 2024-11-10T09:16:38,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/38e6947bacf1455199532c7d0a93acfa to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/38e6947bacf1455199532c7d0a93acfa 2024-11-10T09:16:38,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e878c9edc1f044c8b42d2be70fccfa78 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/e878c9edc1f044c8b42d2be70fccfa78 2024-11-10T09:16:38,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bf1d0960d74a423bb8410c422ac604e1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bf1d0960d74a423bb8410c422ac604e1 2024-11-10T09:16:38,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/85df83e024e34ae39053b863b9170be8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/85df83e024e34ae39053b863b9170be8 2024-11-10T09:16:38,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/3374a8ac22004991872c04a4aeeceac0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/3374a8ac22004991872c04a4aeeceac0 2024-11-10T09:16:38,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/b2f1c6ffd5ee4da197d7ef44000c161a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/b2f1c6ffd5ee4da197d7ef44000c161a 2024-11-10T09:16:38,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6c35adf79882445ca00a07d9cb11bf7f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6c35adf79882445ca00a07d9cb11bf7f 2024-11-10T09:16:38,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/7d9b8606e6514296808457c6b5957f70 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/7d9b8606e6514296808457c6b5957f70 2024-11-10T09:16:38,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/90e96d7a36dc4bbf8153833f2243b1ca to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/90e96d7a36dc4bbf8153833f2243b1ca 2024-11-10T09:16:38,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bce74014ea76462e8fc98ed43072e0e9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/bce74014ea76462e8fc98ed43072e0e9 2024-11-10T09:16:38,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/29ee56619ced4a59b4ade67ccec6ca69 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/29ee56619ced4a59b4ade67ccec6ca69 2024-11-10T09:16:38,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/0eb05834e7aa4c4587a9f9fb1f5b20a9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/0eb05834e7aa4c4587a9f9fb1f5b20a9 2024-11-10T09:16:38,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/43646280b52a45468938001859c786d6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/43646280b52a45468938001859c786d6 2024-11-10T09:16:38,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fcb843e46fb2434897678da7a8aeabee to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/fcb843e46fb2434897678da7a8aeabee 2024-11-10T09:16:38,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5df41a5cb52f4cce8353895811236cdc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/5df41a5cb52f4cce8353895811236cdc 2024-11-10T09:16:38,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/42b4b13592d34d90b58adbec2db02f46 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/42b4b13592d34d90b58adbec2db02f46 2024-11-10T09:16:38,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/289a56a83cc84676a21373ad02f47914 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/289a56a83cc84676a21373ad02f47914 2024-11-10T09:16:38,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/a571df21cff34e36857ee248c17948f2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/a571df21cff34e36857ee248c17948f2 2024-11-10T09:16:38,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f9712b59d46493391afccdfbc2d6682 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f9712b59d46493391afccdfbc2d6682 2024-11-10T09:16:38,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6d889531306a48f09e496b58c5cc4555 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/6d889531306a48f09e496b58c5cc4555 2024-11-10T09:16:38,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/d6aa23abd2e74b6c854d6c2b9e39a7e8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/d6aa23abd2e74b6c854d6c2b9e39a7e8 2024-11-10T09:16:38,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f39d48639e8410da9d2ca567954ef95 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/1f39d48639e8410da9d2ca567954ef95 2024-11-10T09:16:38,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/c485c1edb98c43adb08272d4099be449 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/c485c1edb98c43adb08272d4099be449 2024-11-10T09:16:38,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/081fdeac543f47029357745da3c6709c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/081fdeac543f47029357745da3c6709c 2024-11-10T09:16:38,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/2f1812c7620f45069a193210d7fcb2cf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/2f1812c7620f45069a193210d7fcb2cf 2024-11-10T09:16:38,420 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/recovered.edits/639.seqid, newMaxSeqId=639, maxSeqId=1 2024-11-10T09:16:38,422 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. 2024-11-10T09:16:38,422 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for bfb4913e15c1cd99308737d49021ea5d: Waiting for close lock at 1731230196329Running coprocessor pre-close hooks at 1731230196329Disabling compacts and flushes for region at 1731230196329Disabling writes for close at 1731230196937 (+608 ms)Obtaining lock to block concurrent updates at 1731230196937Preparing flush snapshotting stores in bfb4913e15c1cd99308737d49021ea5d at 1731230196937Finished memstore snapshotting TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731230196938 (+1 ms)Flushing stores of TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d. at 1731230196939 (+1 ms)Flushing bfb4913e15c1cd99308737d49021ea5d/A: creating writer at 1731230196939Flushing bfb4913e15c1cd99308737d49021ea5d/A: appending metadata at 1731230196946 (+7 ms)Flushing bfb4913e15c1cd99308737d49021ea5d/A: closing flushed file at 1731230196946Flushing bfb4913e15c1cd99308737d49021ea5d/B: creating writer at 1731230197373 (+427 ms)Flushing bfb4913e15c1cd99308737d49021ea5d/B: appending metadata at 1731230197378 (+5 ms)Flushing bfb4913e15c1cd99308737d49021ea5d/B: closing flushed file at 1731230197378Flushing bfb4913e15c1cd99308737d49021ea5d/C: creating writer at 1731230197795 (+417 ms)Flushing bfb4913e15c1cd99308737d49021ea5d/C: appending metadata at 1731230197797 (+2 ms)Flushing bfb4913e15c1cd99308737d49021ea5d/C: closing flushed file at 1731230197798 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46dbfab0: reopening flushed file at 1731230198207 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c7c0871: reopening flushed file at 1731230198213 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55bfa0c1: reopening flushed file at 1731230198219 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for bfb4913e15c1cd99308737d49021ea5d in 1289ms, sequenceid=636, compaction requested=true at 1731230198226 (+7 ms)Writing region close event to WAL at 1731230198416 (+190 ms)Running coprocessor post-close hooks at 1731230198421 (+5 ms)Closed at 1731230198422 (+1 ms) 2024-11-10T09:16:38,425 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:38,426 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=bfb4913e15c1cd99308737d49021ea5d, regionState=CLOSED 2024-11-10T09:16:38,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:16:38,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T09:16:38,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure bfb4913e15c1cd99308737d49021ea5d, server=802346a334f5,38513,1731230168538 in 2.2590 sec 2024-11-10T09:16:38,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-10T09:16:38,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfb4913e15c1cd99308737d49021ea5d, UNASSIGN in 2.2650 sec 2024-11-10T09:16:38,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T09:16:38,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2720 sec 2024-11-10T09:16:38,437 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230198437"}]},"ts":"1731230198437"} 2024-11-10T09:16:38,439 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:16:38,439 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:16:38,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2860 sec 2024-11-10T09:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:16:40,318 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:40,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:16:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,327 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:16:40,329 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,333 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:40,338 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/recovered.edits] 2024-11-10T09:16:40,341 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/386504bec3db4a2f945b693f6c609b98 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/386504bec3db4a2f945b693f6c609b98 2024-11-10T09:16:40,343 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bdec02c93c574ded80ed9826bebffa71 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/bdec02c93c574ded80ed9826bebffa71 2024-11-10T09:16:40,344 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/beeaa2e2f15e48dd8d6c840fc5532ba9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/beeaa2e2f15e48dd8d6c840fc5532ba9 2024-11-10T09:16:40,346 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f573f7c9500b4376aecaf63de8e9a1f3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f573f7c9500b4376aecaf63de8e9a1f3 2024-11-10T09:16:40,347 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f58fbc03da4e410e8b080430b70dcdac to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/A/f58fbc03da4e410e8b080430b70dcdac 2024-11-10T09:16:40,349 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1cfc67767e594c8293f806761a219dfa to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/1cfc67767e594c8293f806761a219dfa 2024-11-10T09:16:40,350 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6daf55c1a9004a96822862dcb3672c23 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/6daf55c1a9004a96822862dcb3672c23 2024-11-10T09:16:40,352 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7bf58eb853784eca853854774e6e4f30 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/7bf58eb853784eca853854774e6e4f30 2024-11-10T09:16:40,353 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/b0312eaa42594a3cacf2d0a33a492883 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/b0312eaa42594a3cacf2d0a33a492883 2024-11-10T09:16:40,355 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d2aea5dfc47e4efabeca25aef5af8113 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/B/d2aea5dfc47e4efabeca25aef5af8113 2024-11-10T09:16:40,358 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/04339bc28a59432e9c71672733c260c9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/04339bc28a59432e9c71672733c260c9 2024-11-10T09:16:40,360 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/25446cb7e7ab4c43b4215ef62eae01ad to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/25446cb7e7ab4c43b4215ef62eae01ad 2024-11-10T09:16:40,362 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/35a28d6468424f98ba7ccbfd866a1b6a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/35a28d6468424f98ba7ccbfd866a1b6a 2024-11-10T09:16:40,364 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/500791b5681b45999021ee6db47090b0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/500791b5681b45999021ee6db47090b0 2024-11-10T09:16:40,366 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/59bf7e34b841424fa779c9360371b6b5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/C/59bf7e34b841424fa779c9360371b6b5 2024-11-10T09:16:40,374 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/recovered.edits/639.seqid to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d/recovered.edits/639.seqid 2024-11-10T09:16:40,374 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/bfb4913e15c1cd99308737d49021ea5d 2024-11-10T09:16:40,374 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:16:40,380 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T09:16:40,391 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:16:40,425 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:16:40,426 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,426 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:16:40,427 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230200426"}]},"ts":"9223372036854775807"} 2024-11-10T09:16:40,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:16:40,431 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bfb4913e15c1cd99308737d49021ea5d, NAME => 'TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:16:40,432 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:16:40,432 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230200432"}]},"ts":"9223372036854775807"} 2024-11-10T09:16:40,435 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:16:40,436 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 114 msec 2024-11-10T09:16:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:16:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:16:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:16:40,649 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:40,664 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;802346a334f5:38513-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_381167618_22 at /127.0.0.1:55438 [Waiting for operation #290] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_381167618_22 at /127.0.0.1:56488 [Waiting for operation #299] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-704730897_22 at /127.0.0.1:51934 [Waiting for operation #123] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=760 (was 663) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2748 (was 3516) 2024-11-10T09:16:40,676 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=760, ProcessCount=11, AvailableMemoryMB=2747 2024-11-10T09:16:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:16:40,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:16:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:40,686 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:16:40,687 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:40,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-10T09:16:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T09:16:40,688 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:16:40,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741951_1127 (size=1061) 2024-11-10T09:16:40,699 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:16:40,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741952_1128 (size=53) 2024-11-10T09:16:40,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:40,719 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 83bc899d24e43ccf5880b01ce9dff713, disabling compactions & flushes 2024-11-10T09:16:40,719 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:40,719 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:40,719 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. after waiting 0 ms 2024-11-10T09:16:40,719 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:40,719 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:40,719 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 83bc899d24e43ccf5880b01ce9dff713: Waiting for close lock at 1731230200719Disabling compacts and flushes for region at 1731230200719Disabling writes for close at 1731230200719Writing region close event to WAL at 1731230200719Closed at 1731230200719 2024-11-10T09:16:40,721 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:16:40,722 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230200721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230200721"}]},"ts":"1731230200721"} 2024-11-10T09:16:40,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:16:40,726 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:16:40,727 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230200726"}]},"ts":"1731230200726"} 2024-11-10T09:16:40,729 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:16:40,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, ASSIGN}] 2024-11-10T09:16:40,730 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, ASSIGN 2024-11-10T09:16:40,731 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, ASSIGN; state=OFFLINE, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=false 2024-11-10T09:16:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T09:16:40,882 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:40,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, ASSIGN because future has completed 2024-11-10T09:16:40,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:16:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T09:16:41,052 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,052 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:16:41,053 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,053 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:41,053 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,053 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,055 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,056 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:41,057 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83bc899d24e43ccf5880b01ce9dff713 columnFamilyName A 2024-11-10T09:16:41,057 DEBUG [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:41,058 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(327): Store=83bc899d24e43ccf5880b01ce9dff713/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:41,058 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,059 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:41,060 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83bc899d24e43ccf5880b01ce9dff713 columnFamilyName B 2024-11-10T09:16:41,060 DEBUG [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:41,060 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(327): Store=83bc899d24e43ccf5880b01ce9dff713/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:41,060 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,061 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:41,062 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83bc899d24e43ccf5880b01ce9dff713 columnFamilyName C 2024-11-10T09:16:41,062 DEBUG [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:41,062 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(327): Store=83bc899d24e43ccf5880b01ce9dff713/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:41,063 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,063 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,064 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,064 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,065 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,065 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,065 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:16:41,066 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,068 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:16:41,069 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 83bc899d24e43ccf5880b01ce9dff713; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61504435, jitterRate=-0.08351249992847443}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:16:41,069 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,069 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 83bc899d24e43ccf5880b01ce9dff713: Running coprocessor pre-open hook at 1731230201053Writing region info on filesystem at 1731230201053Initializing all the Stores at 1731230201054 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230201054Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230201054Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230201055 (+1 ms)Cleaning up temporary data from old regions at 1731230201065 (+10 ms)Running coprocessor post-open hooks at 1731230201069 (+4 ms)Region opened successfully at 1731230201069 2024-11-10T09:16:41,070 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., pid=32, masterSystemTime=1731230201048 2024-11-10T09:16:41,072 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,072 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,073 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:41,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:16:41,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T09:16:41,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 in 181 msec 2024-11-10T09:16:41,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-10T09:16:41,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, ASSIGN in 349 msec 2024-11-10T09:16:41,081 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:16:41,082 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230201081"}]},"ts":"1731230201081"} 2024-11-10T09:16:41,085 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:16:41,086 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:16:41,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 407 msec 2024-11-10T09:16:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T09:16:41,319 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:41,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dac73a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,326 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d5d3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,330 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:16:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:16:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-10T09:16:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:16:41,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:16:41,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:16:41,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:16:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T09:16:41,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741953_1129 (size=1097) 2024-11-10T09:16:41,362 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T09:16:41,362 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T09:16:41,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:16:41,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, REOPEN/MOVE}] 2024-11-10T09:16:41,378 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, REOPEN/MOVE 2024-11-10T09:16:41,380 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:41,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, REOPEN/MOVE because future has completed 2024-11-10T09:16:41,383 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:16:41,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:16:41,538 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,538 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:16:41,538 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 83bc899d24e43ccf5880b01ce9dff713, disabling compactions & flushes 2024-11-10T09:16:41,538 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,538 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,538 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. after waiting 0 ms 2024-11-10T09:16:41,538 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,542 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T09:16:41,543 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,543 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 83bc899d24e43ccf5880b01ce9dff713: Waiting for close lock at 1731230201538Running coprocessor pre-close hooks at 1731230201538Disabling compacts and flushes for region at 1731230201538Disabling writes for close at 1731230201538Writing region close event to WAL at 1731230201539 (+1 ms)Running coprocessor post-close hooks at 1731230201543 (+4 ms)Closed at 1731230201543 2024-11-10T09:16:41,543 WARN [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 83bc899d24e43ccf5880b01ce9dff713 to self. 2024-11-10T09:16:41,545 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,546 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=CLOSED 2024-11-10T09:16:41,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:16:41,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-10T09:16:41,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 in 166 msec 2024-11-10T09:16:41,552 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, REOPEN/MOVE; state=CLOSED, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=true 2024-11-10T09:16:41,702 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:41,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, REOPEN/MOVE because future has completed 2024-11-10T09:16:41,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:16:41,861 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,861 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:16:41,861 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,862 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:16:41,862 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,862 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,865 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,866 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:41,872 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83bc899d24e43ccf5880b01ce9dff713 columnFamilyName A 2024-11-10T09:16:41,875 DEBUG [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:41,876 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(327): Store=83bc899d24e43ccf5880b01ce9dff713/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:41,877 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,877 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:41,878 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83bc899d24e43ccf5880b01ce9dff713 columnFamilyName B 2024-11-10T09:16:41,878 DEBUG [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:41,879 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(327): Store=83bc899d24e43ccf5880b01ce9dff713/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:41,879 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,880 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:16:41,880 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83bc899d24e43ccf5880b01ce9dff713 columnFamilyName C 2024-11-10T09:16:41,880 DEBUG [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:41,880 INFO [StoreOpener-83bc899d24e43ccf5880b01ce9dff713-1 {}] regionserver.HStore(327): Store=83bc899d24e43ccf5880b01ce9dff713/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:16:41,881 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,881 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,881 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,882 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,883 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,883 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,884 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:16:41,885 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,886 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 83bc899d24e43ccf5880b01ce9dff713; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63874968, jitterRate=-0.04818880558013916}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:16:41,886 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,888 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 83bc899d24e43ccf5880b01ce9dff713: Running coprocessor pre-open hook at 1731230201862Writing region info on filesystem at 1731230201862Initializing all the Stores at 1731230201863 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230201863Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230201863Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230201864 (+1 ms)Cleaning up temporary data from old regions at 1731230201883 (+19 ms)Running coprocessor post-open hooks at 1731230201886 (+3 ms)Region opened successfully at 1731230201887 (+1 ms) 2024-11-10T09:16:41,888 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., pid=37, masterSystemTime=1731230201857 2024-11-10T09:16:41,891 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,891 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:41,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=OPEN, openSeqNum=5, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:16:41,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:16:41,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-10T09:16:41,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 in 189 msec 2024-11-10T09:16:41,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-10T09:16:41,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, REOPEN/MOVE in 520 msec 2024-11-10T09:16:41,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-10T09:16:41,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-11-10T09:16:41,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 558 msec 2024-11-10T09:16:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-10T09:16:41,907 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:41,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632b0d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,908 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d16f94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c1db30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,912 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d92e8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,913 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329ed148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58199921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,916 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2436d8d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,917 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200c3196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f37e1f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,919 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d8cf7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cf21c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37cffe5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb05800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bab58c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12959706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b108628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,928 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511b10f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6ee02c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f1c931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4935f1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626a197f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:16:41,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:16:41,937 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:16:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:16:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:16:41,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ec7582, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:16:41,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:16:41,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:16:41,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:16:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:16:41,940 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,940 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,940 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,940 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,941 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,942 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,942 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,944 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,944 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:16:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-10T09:16:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:16:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:16:41,949 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T09:16:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,950 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:41,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:41,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:16:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:16:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:41,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:42,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cfb8217c879645fdb3b7a3b44a9ea729_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230201970/Put/seqid=0 2024-11-10T09:16:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741954_1130 (size=14594) 2024-11-10T09:16:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T09:16:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230262049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230262052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230202040, max timestamp=1731230202053Num uniques 859; 2024-11-10T09:16:42,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230202040, max timestamp=1731230202053Num uniques 859; 2024-11-10T09:16:42,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:42,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230202040, max timestamp=1731230202053Num uniques 859; 2024-11-10T09:16:42,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:16:42,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:42,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230262136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230262141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230262142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230262168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230262168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230262248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230262258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230262258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T09:16:42,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230262378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230262378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:42,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cfb8217c879645fdb3b7a3b44a9ea729_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cfb8217c879645fdb3b7a3b44a9ea729_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:42,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/c173208aa56a433496837422884fae08, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:42,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:42,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:42,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/c173208aa56a433496837422884fae08 is 175, key is test_row_0/A:col10/1731230201970/Put/seqid=0 2024-11-10T09:16:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741955_1131 (size=39549) 2024-11-10T09:16:42,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/c173208aa56a433496837422884fae08 2024-11-10T09:16:42,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230262467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230262467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230262468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/1121b1c952a941c183111ea65bc3e1fc is 50, key is test_row_0/B:col10/1731230201970/Put/seqid=0 2024-11-10T09:16:42,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741956_1132 (size=12001) 2024-11-10T09:16:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T09:16:42,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:42,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:42,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230262687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230262697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:42,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:42,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230262777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230262777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230262778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:42,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:42,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:42,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:42,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/1121b1c952a941c183111ea65bc3e1fc 2024-11-10T09:16:43,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/d13319252f044a6fbb190c02e88eb43b is 50, key is test_row_0/C:col10/1731230201970/Put/seqid=0 2024-11-10T09:16:43,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741957_1133 (size=12001) 2024-11-10T09:16:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T09:16:43,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230263208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230263208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:43,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230263287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:43,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230263287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:43,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230263297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:43,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:43,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/d13319252f044a6fbb190c02e88eb43b 2024-11-10T09:16:43,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/c173208aa56a433496837422884fae08 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08 2024-11-10T09:16:43,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08, entries=200, sequenceid=15, filesize=38.6 K 2024-11-10T09:16:43,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/1121b1c952a941c183111ea65bc3e1fc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/1121b1c952a941c183111ea65bc3e1fc 2024-11-10T09:16:43,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/1121b1c952a941c183111ea65bc3e1fc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:16:43,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/d13319252f044a6fbb190c02e88eb43b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/d13319252f044a6fbb190c02e88eb43b 2024-11-10T09:16:43,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:43,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/d13319252f044a6fbb190c02e88eb43b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:16:43,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 83bc899d24e43ccf5880b01ce9dff713 in 1544ms, sequenceid=15, compaction requested=false 2024-11-10T09:16:43,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:43,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T09:16:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,665 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T09:16:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:43,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103f1f62b641d9430fb62ed138eb8f3bf1_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230202135/Put/seqid=0 2024-11-10T09:16:43,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741958_1134 (size=12154) 2024-11-10T09:16:43,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,736 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103f1f62b641d9430fb62ed138eb8f3bf1_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103f1f62b641d9430fb62ed138eb8f3bf1_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/b94430202ff44170bda51b885ee55168, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/b94430202ff44170bda51b885ee55168 is 175, key is test_row_0/A:col10/1731230202135/Put/seqid=0 2024-11-10T09:16:43,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741959_1135 (size=30955) 2024-11-10T09:16:43,765 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/b94430202ff44170bda51b885ee55168 2024-11-10T09:16:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/4636dbfc66c84b2bb009b66d196d1494 is 50, key is test_row_0/B:col10/1731230202135/Put/seqid=0 2024-11-10T09:16:43,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741960_1136 (size=12001) 2024-11-10T09:16:43,845 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/4636dbfc66c84b2bb009b66d196d1494 2024-11-10T09:16:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/e374cb2a201c4a8c946baa1199f028b5 is 50, key is test_row_0/C:col10/1731230202135/Put/seqid=0 2024-11-10T09:16:43,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741961_1137 (size=12001) 2024-11-10T09:16:43,908 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/e374cb2a201c4a8c946baa1199f028b5 2024-11-10T09:16:43,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/b94430202ff44170bda51b885ee55168 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168 2024-11-10T09:16:43,938 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168, entries=150, sequenceid=51, filesize=30.2 K 2024-11-10T09:16:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/4636dbfc66c84b2bb009b66d196d1494 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4636dbfc66c84b2bb009b66d196d1494 2024-11-10T09:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,964 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4636dbfc66c84b2bb009b66d196d1494, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T09:16:43,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/e374cb2a201c4a8c946baa1199f028b5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/e374cb2a201c4a8c946baa1199f028b5 2024-11-10T09:16:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,991 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:16:43,991 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/e374cb2a201c4a8c946baa1199f028b5, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T09:16:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,993 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 83bc899d24e43ccf5880b01ce9dff713 in 328ms, sequenceid=51, compaction requested=false 2024-11-10T09:16:43,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:43,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-10T09:16:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-10T09:16:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T09:16:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0450 sec 2024-11-10T09:16:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.0580 sec 2024-11-10T09:16:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T09:16:44,098 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:44,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T09:16:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T09:16:44,103 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:44,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T09:16:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T09:16:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T09:16:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T09:16:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T09:16:44,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-10T09:16:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 167 msec 2024-11-10T09:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T09:16:44,418 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T09:16:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:44,428 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:44,429 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:44,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111061f54f5e513b4ff0b5866f2b7fda1700_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741962_1138 (size=21918) 2024-11-10T09:16:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111061f54f5e513b4ff0b5866f2b7fda1700_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061f54f5e513b4ff0b5866f2b7fda1700_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b270949f9df4d6db6c1106c8ff4e676, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:44,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b270949f9df4d6db6c1106c8ff4e676 is 175, key is test_row_0/A:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741963_1139 (size=65319) 2024-11-10T09:16:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731230204412, max timestamp=1731230204566Num uniques 859; 2024-11-10T09:16:44,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731230204412, max timestamp=1731230204566Num uniques 859; 2024-11-10T09:16:44,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T09:16:44,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T09:16:44,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731230204412, max timestamp=1731230204566Num uniques 859; 2024-11-10T09:16:44,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T09:16:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:44,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230264578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230264580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230264583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230264583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230264590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230264698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230264698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230264698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230264699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230264708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:44,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:44,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:44,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:44,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:44,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230264908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230264908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230264909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230264908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230264928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:44,953 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b270949f9df4d6db6c1106c8ff4e676 2024-11-10T09:16:44,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5acae3625d274e88849e30a437ec052b is 50, key is test_row_0/B:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741964_1140 (size=9657) 2024-11-10T09:16:45,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230265229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230265229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230265229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230265230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230265248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5acae3625d274e88849e30a437ec052b 2024-11-10T09:16:45,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/25e3531dedca419fa00ddd4a6c2c7ac8 is 50, key is test_row_0/C:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:45,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741965_1141 (size=9657) 2024-11-10T09:16:45,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:45,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230265744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230265750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230265750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230265761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230265769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:45,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/25e3531dedca419fa00ddd4a6c2c7ac8 2024-11-10T09:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b270949f9df4d6db6c1106c8ff4e676 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676 2024-11-10T09:16:45,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676, entries=350, sequenceid=62, filesize=63.8 K 2024-11-10T09:16:45,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5acae3625d274e88849e30a437ec052b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5acae3625d274e88849e30a437ec052b 2024-11-10T09:16:45,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5acae3625d274e88849e30a437ec052b, entries=100, sequenceid=62, filesize=9.4 K 2024-11-10T09:16:45,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/25e3531dedca419fa00ddd4a6c2c7ac8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/25e3531dedca419fa00ddd4a6c2c7ac8 2024-11-10T09:16:45,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/25e3531dedca419fa00ddd4a6c2c7ac8, entries=100, sequenceid=62, filesize=9.4 K 2024-11-10T09:16:45,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=161.02 KB/164880 for 83bc899d24e43ccf5880b01ce9dff713 in 1588ms, sequenceid=62, compaction requested=true 2024-11-10T09:16:45,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:45,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:45,994 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:45,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:45,994 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:45,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:45,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:45,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:45,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:45,995 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:45,996 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:45,996 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,996 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/1121b1c952a941c183111ea65bc3e1fc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4636dbfc66c84b2bb009b66d196d1494, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5acae3625d274e88849e30a437ec052b] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=32.9 K 2024-11-10T09:16:45,996 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:45,997 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:45,997 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,997 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=132.6 K 2024-11-10T09:16:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:45,997 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:45,997 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676] 2024-11-10T09:16:45,998 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1121b1c952a941c183111ea65bc3e1fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230201962 2024-11-10T09:16:45,999 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4636dbfc66c84b2bb009b66d196d1494, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230202117 2024-11-10T09:16:45,999 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c173208aa56a433496837422884fae08, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230201962 2024-11-10T09:16:45,999 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b94430202ff44170bda51b885ee55168, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230202117 2024-11-10T09:16:45,999 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5acae3625d274e88849e30a437ec052b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731230204329 2024-11-10T09:16:46,000 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b270949f9df4d6db6c1106c8ff4e676, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731230204322 2024-11-10T09:16:46,026 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:46,035 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:46,035 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/f87896ab9d814d2283a8d2db5e7514d4 is 50, key is test_row_0/B:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:46,043 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110eb3e0483db854175b1ca94930aeef664_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:46,047 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110eb3e0483db854175b1ca94930aeef664_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:46,047 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb3e0483db854175b1ca94930aeef664_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:46,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:16:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:46,153 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=161.02 KB heapSize=399.05 KB 2024-11-10T09:16:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:46,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741966_1142 (size=12104) 2024-11-10T09:16:46,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741967_1143 (size=4469) 2024-11-10T09:16:46,196 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#123 average throughput is 0.14 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:46,197 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/4e399a0686b34ed4824085bd70eb638a is 175, key is test_row_0/A:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:46,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fcb2e1367fc74e289e947248b48f7ce4_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230204579/Put/seqid=0 2024-11-10T09:16:46,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741968_1144 (size=31165) 2024-11-10T09:16:46,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741969_1145 (size=12154) 2024-11-10T09:16:46,566 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/f87896ab9d814d2283a8d2db5e7514d4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f87896ab9d814d2283a8d2db5e7514d4 2024-11-10T09:16:46,573 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into f87896ab9d814d2283a8d2db5e7514d4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:46,573 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:46,573 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230205994; duration=0sec 2024-11-10T09:16:46,573 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:46,573 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:16:46,573 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:46,575 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:46,575 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:16:46,575 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:46,575 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/d13319252f044a6fbb190c02e88eb43b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/e374cb2a201c4a8c946baa1199f028b5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/25e3531dedca419fa00ddd4a6c2c7ac8] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=32.9 K 2024-11-10T09:16:46,576 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d13319252f044a6fbb190c02e88eb43b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230201962 2024-11-10T09:16:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:46,579 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e374cb2a201c4a8c946baa1199f028b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230202117 2024-11-10T09:16:46,581 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e3531dedca419fa00ddd4a6c2c7ac8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731230204329 2024-11-10T09:16:46,607 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:46,608 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/63751e38933a4914b4f1e910163100e3 is 50, key is test_row_0/C:col10/1731230204403/Put/seqid=0 2024-11-10T09:16:46,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741970_1146 (size=12104) 2024-11-10T09:16:46,632 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/63751e38933a4914b4f1e910163100e3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/63751e38933a4914b4f1e910163100e3 2024-11-10T09:16:46,640 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into 63751e38933a4914b4f1e910163100e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:46,640 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:46,640 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230205995; duration=0sec 2024-11-10T09:16:46,640 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:46,640 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:16:46,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/4e399a0686b34ed4824085bd70eb638a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4e399a0686b34ed4824085bd70eb638a 2024-11-10T09:16:46,674 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 4e399a0686b34ed4824085bd70eb638a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:46,674 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:46,674 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230205994; duration=0sec 2024-11-10T09:16:46,675 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:46,675 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:16:46,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:46,681 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fcb2e1367fc74e289e947248b48f7ce4_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fcb2e1367fc74e289e947248b48f7ce4_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/50b82b526ed0403eaa46f4b565543bc6, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:46,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/50b82b526ed0403eaa46f4b565543bc6 is 175, key is test_row_0/A:col10/1731230204579/Put/seqid=0 2024-11-10T09:16:46,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741971_1147 (size=30955) 2024-11-10T09:16:46,716 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/50b82b526ed0403eaa46f4b565543bc6 2024-11-10T09:16:46,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/c3e256aa75a5477bb88e1b3cba03b0bb is 50, key is test_row_0/B:col10/1731230204579/Put/seqid=0 2024-11-10T09:16:46,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741972_1148 (size=12001) 2024-11-10T09:16:46,760 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/c3e256aa75a5477bb88e1b3cba03b0bb 2024-11-10T09:16:46,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:46,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230266785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/51b7863cc4f045b48a024a308ae3a7ec is 50, key is test_row_0/C:col10/1731230204579/Put/seqid=0 2024-11-10T09:16:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230266789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230266788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230266790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230266792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,796 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 83bc899d24e43ccf5880b01ce9dff713 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741973_1149 (size=12001) 2024-11-10T09:16:46,825 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/51b7863cc4f045b48a024a308ae3a7ec 2024-11-10T09:16:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/50b82b526ed0403eaa46f4b565543bc6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6 2024-11-10T09:16:46,845 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6, entries=150, sequenceid=89, filesize=30.2 K 2024-11-10T09:16:46,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/c3e256aa75a5477bb88e1b3cba03b0bb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/c3e256aa75a5477bb88e1b3cba03b0bb 2024-11-10T09:16:46,862 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/c3e256aa75a5477bb88e1b3cba03b0bb, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T09:16:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/51b7863cc4f045b48a024a308ae3a7ec as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/51b7863cc4f045b48a024a308ae3a7ec 2024-11-10T09:16:46,872 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/51b7863cc4f045b48a024a308ae3a7ec, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T09:16:46,889 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~399 KB/408576, currentSize=53.67 KB/54960 for 83bc899d24e43ccf5880b01ce9dff713 in 735ms, sequenceid=89, compaction requested=false 2024-11-10T09:16:46,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:46,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:46,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T09:16:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T09:16:46,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T09:16:46,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4700 sec 2024-11-10T09:16:46,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.4850 sec 2024-11-10T09:16:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:46,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:16:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:46,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111038b99edae9e24026b17c10cae93fc0c7_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:46,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741974_1150 (size=12154) 2024-11-10T09:16:46,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:46,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230266978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,988 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111038b99edae9e24026b17c10cae93fc0c7_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111038b99edae9e24026b17c10cae93fc0c7_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:46,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/733fc9bbb6264bb6abeea500da2b144d, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:46,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230266984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/733fc9bbb6264bb6abeea500da2b144d is 175, key is test_row_0/A:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230266985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230266986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741975_1151 (size=30955) 2024-11-10T09:16:47,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/733fc9bbb6264bb6abeea500da2b144d 2024-11-10T09:16:47,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/ae7a165a69df4d64ab8a16de396abc97 is 50, key is test_row_0/B:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:47,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741976_1152 (size=12001) 2024-11-10T09:16:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230267098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230267098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230267107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230267110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230267308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230267308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230267317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230267317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/ae7a165a69df4d64ab8a16de396abc97 2024-11-10T09:16:47,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/36ab9ce24b5d498aa5000cff3265938a is 50, key is test_row_0/C:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:47,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741977_1153 (size=12001) 2024-11-10T09:16:47,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/36ab9ce24b5d498aa5000cff3265938a 2024-11-10T09:16:47,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/733fc9bbb6264bb6abeea500da2b144d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d 2024-11-10T09:16:47,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d, entries=150, sequenceid=105, filesize=30.2 K 2024-11-10T09:16:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/ae7a165a69df4d64ab8a16de396abc97 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ae7a165a69df4d64ab8a16de396abc97 2024-11-10T09:16:47,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ae7a165a69df4d64ab8a16de396abc97, entries=150, sequenceid=105, filesize=11.7 K 2024-11-10T09:16:47,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/36ab9ce24b5d498aa5000cff3265938a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/36ab9ce24b5d498aa5000cff3265938a 2024-11-10T09:16:47,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/36ab9ce24b5d498aa5000cff3265938a, entries=150, sequenceid=105, filesize=11.7 K 2024-11-10T09:16:47,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 83bc899d24e43ccf5880b01ce9dff713 in 655ms, sequenceid=105, compaction requested=true 2024-11-10T09:16:47,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:47,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:47,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:47,569 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:47,569 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:47,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:47,570 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:47,570 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:47,570 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:47,570 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4e399a0686b34ed4824085bd70eb638a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=90.9 K 2024-11-10T09:16:47,570 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:47,570 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4e399a0686b34ed4824085bd70eb638a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d] 2024-11-10T09:16:47,571 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:47,571 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:47,571 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:47,571 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f87896ab9d814d2283a8d2db5e7514d4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/c3e256aa75a5477bb88e1b3cba03b0bb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ae7a165a69df4d64ab8a16de396abc97] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=35.3 K 2024-11-10T09:16:47,572 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f87896ab9d814d2283a8d2db5e7514d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731230202117 2024-11-10T09:16:47,572 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e399a0686b34ed4824085bd70eb638a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731230202117 2024-11-10T09:16:47,572 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e256aa75a5477bb88e1b3cba03b0bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731230204578 2024-11-10T09:16:47,572 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b82b526ed0403eaa46f4b565543bc6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731230204578 2024-11-10T09:16:47,573 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ae7a165a69df4d64ab8a16de396abc97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230206774 2024-11-10T09:16:47,573 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733fc9bbb6264bb6abeea500da2b144d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230206774 2024-11-10T09:16:47,589 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:47,597 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110aaa8771e9e094afb9a5f4e5d14a052d8_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:47,598 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:47,599 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/fa9e30ea8dd44e22ab866e22bdf382ab is 50, key is test_row_0/B:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:47,600 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110aaa8771e9e094afb9a5f4e5d14a052d8_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:47,600 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aaa8771e9e094afb9a5f4e5d14a052d8_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:47,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741978_1154 (size=12207) 2024-11-10T09:16:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:47,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:16:47,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230206949, max timestamp=1731230207620Num uniques 859; 2024-11-10T09:16:47,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:16:47,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741979_1155 (size=4469) 2024-11-10T09:16:47,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:47,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:47,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:47,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:47,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:47,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:47,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111010b7f4be771045158e127c0fd0f39391_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230207619/Put/seqid=0 2024-11-10T09:16:47,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741980_1156 (size=14594) 2024-11-10T09:16:47,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:47,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111010b7f4be771045158e127c0fd0f39391_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111010b7f4be771045158e127c0fd0f39391_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:47,657 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/3781ef5da5eb4f158b5a796d6bdf1850, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:47,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/3781ef5da5eb4f158b5a796d6bdf1850 is 175, key is test_row_0/A:col10/1731230207619/Put/seqid=0 2024-11-10T09:16:47,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230267662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230267659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230267662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230267664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741981_1157 (size=39549) 2024-11-10T09:16:47,674 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/3781ef5da5eb4f158b5a796d6bdf1850 2024-11-10T09:16:47,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/e5cfcb06aa994703b7f52e30e6c2ffdd is 50, key is test_row_0/B:col10/1731230207619/Put/seqid=0 2024-11-10T09:16:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741982_1158 (size=12001) 2024-11-10T09:16:47,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/e5cfcb06aa994703b7f52e30e6c2ffdd 2024-11-10T09:16:47,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/831c1aac2169420fa3ff40ee2cf443ae is 50, key is test_row_0/C:col10/1731230207619/Put/seqid=0 2024-11-10T09:16:47,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741983_1159 (size=12001) 2024-11-10T09:16:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230267777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230267777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230267778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230267778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230267987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230267987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230267987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230267988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,025 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#132 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:48,027 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/92bf309a68f24de4ae0b6abc7fcc46ba is 175, key is test_row_0/A:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:48,027 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/fa9e30ea8dd44e22ab866e22bdf382ab as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/fa9e30ea8dd44e22ab866e22bdf382ab 2024-11-10T09:16:48,039 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into fa9e30ea8dd44e22ab866e22bdf382ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:48,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:48,039 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230207568; duration=0sec 2024-11-10T09:16:48,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:48,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:16:48,040 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:48,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:48,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:16:48,041 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:48,041 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/63751e38933a4914b4f1e910163100e3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/51b7863cc4f045b48a024a308ae3a7ec, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/36ab9ce24b5d498aa5000cff3265938a] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=35.3 K 2024-11-10T09:16:48,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 63751e38933a4914b4f1e910163100e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731230202117 2024-11-10T09:16:48,042 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 51b7863cc4f045b48a024a308ae3a7ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731230204578 2024-11-10T09:16:48,042 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ab9ce24b5d498aa5000cff3265938a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230206774 2024-11-10T09:16:48,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741984_1160 (size=31161) 2024-11-10T09:16:48,057 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/92bf309a68f24de4ae0b6abc7fcc46ba as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/92bf309a68f24de4ae0b6abc7fcc46ba 2024-11-10T09:16:48,058 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:48,059 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/b42c3761280742f3b633d691cb11a846 is 50, key is test_row_0/C:col10/1731230206774/Put/seqid=0 2024-11-10T09:16:48,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741985_1161 (size=12207) 2024-11-10T09:16:48,068 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 92bf309a68f24de4ae0b6abc7fcc46ba(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:48,068 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:48,068 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230207568; duration=0sec 2024-11-10T09:16:48,068 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:48,068 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:16:48,076 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/b42c3761280742f3b633d691cb11a846 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b42c3761280742f3b633d691cb11a846 2024-11-10T09:16:48,083 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into b42c3761280742f3b633d691cb11a846(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:48,083 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:48,083 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230207569; duration=0sec 2024-11-10T09:16:48,083 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:48,083 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:16:48,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/831c1aac2169420fa3ff40ee2cf443ae 2024-11-10T09:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/3781ef5da5eb4f158b5a796d6bdf1850 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850 2024-11-10T09:16:48,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850, entries=200, sequenceid=126, filesize=38.6 K 2024-11-10T09:16:48,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/e5cfcb06aa994703b7f52e30e6c2ffdd as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/e5cfcb06aa994703b7f52e30e6c2ffdd 2024-11-10T09:16:48,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/e5cfcb06aa994703b7f52e30e6c2ffdd, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T09:16:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/831c1aac2169420fa3ff40ee2cf443ae as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/831c1aac2169420fa3ff40ee2cf443ae 2024-11-10T09:16:48,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/831c1aac2169420fa3ff40ee2cf443ae, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T09:16:48,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 83bc899d24e43ccf5880b01ce9dff713 in 520ms, sequenceid=126, compaction requested=false 2024-11-10T09:16:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:48,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:48,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:48,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:48,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108499c40f8b094483b9ca80bf9eeb9ab7_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230268331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741986_1162 (size=14794) 2024-11-10T09:16:48,340 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:48,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230268333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230268333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230268335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108499c40f8b094483b9ca80bf9eeb9ab7_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108499c40f8b094483b9ca80bf9eeb9ab7_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:48,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/2eb8737e9214420bb3118f43396f58c6, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:48,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/2eb8737e9214420bb3118f43396f58c6 is 175, key is test_row_0/A:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:48,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741987_1163 (size=39749) 2024-11-10T09:16:48,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230268447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230268447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230268447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230268448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:16:48,597 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:48,598 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T09:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:16:48,599 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:48,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:48,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230268657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230268657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230268658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230268658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:16:48,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:16:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:48,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:48,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:48,779 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/2eb8737e9214420bb3118f43396f58c6 2024-11-10T09:16:48,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/8e6e463db76344a68f962981dbafbcbf is 50, key is test_row_0/B:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:48,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230268807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,813 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 83bc899d24e43ccf5880b01ce9dff713 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:16:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741988_1164 (size=12151) 2024-11-10T09:16:48,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/8e6e463db76344a68f962981dbafbcbf 2024-11-10T09:16:48,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/85a8a4340e784451bc051525e8a04ca7 is 50, key is test_row_0/C:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741989_1165 (size=12151) 2024-11-10T09:16:48,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:16:48,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:48,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:48,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:48,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:48,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:16:48,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230268967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230268978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230268979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:48,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230268980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:16:49,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:49,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:16:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:16:49,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/85a8a4340e784451bc051525e8a04ca7 2024-11-10T09:16:49,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/2eb8737e9214420bb3118f43396f58c6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6 2024-11-10T09:16:49,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6, entries=200, sequenceid=148, filesize=38.8 K 2024-11-10T09:16:49,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/8e6e463db76344a68f962981dbafbcbf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/8e6e463db76344a68f962981dbafbcbf 2024-11-10T09:16:49,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/8e6e463db76344a68f962981dbafbcbf, entries=150, sequenceid=148, filesize=11.9 K 2024-11-10T09:16:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/85a8a4340e784451bc051525e8a04ca7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/85a8a4340e784451bc051525e8a04ca7 2024-11-10T09:16:49,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/85a8a4340e784451bc051525e8a04ca7, entries=150, sequenceid=148, filesize=11.9 K 2024-11-10T09:16:49,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 83bc899d24e43ccf5880b01ce9dff713 in 1013ms, sequenceid=148, compaction requested=true 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:49,317 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:49,317 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:49,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:49,319 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:49,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:49,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:49,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:49,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,320 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,320 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/fa9e30ea8dd44e22ab866e22bdf382ab, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/e5cfcb06aa994703b7f52e30e6c2ffdd, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/8e6e463db76344a68f962981dbafbcbf] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=35.5 K 2024-11-10T09:16:49,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/92bf309a68f24de4ae0b6abc7fcc46ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=107.9 K 2024-11-10T09:16:49,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/92bf309a68f24de4ae0b6abc7fcc46ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6] 2024-11-10T09:16:49,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9e30ea8dd44e22ab866e22bdf382ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230206774 2024-11-10T09:16:49,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92bf309a68f24de4ae0b6abc7fcc46ba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230206774 2024-11-10T09:16:49,321 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e5cfcb06aa994703b7f52e30e6c2ffdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731230206949 2024-11-10T09:16:49,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3781ef5da5eb4f158b5a796d6bdf1850, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731230206949 2024-11-10T09:16:49,321 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6e463db76344a68f962981dbafbcbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731230207659 2024-11-10T09:16:49,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb8737e9214420bb3118f43396f58c6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731230207636 2024-11-10T09:16:49,338 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/6b0fd52a5e8d4e308aaa0835e56f5944 is 50, key is test_row_0/B:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:49,347 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:49,373 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110acb0f4c7f65a4b70965730f0eb5cd61c_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:49,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110acb0f4c7f65a4b70965730f0eb5cd61c_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:49,376 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110acb0f4c7f65a4b70965730f0eb5cd61c_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:49,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:16:49,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,379 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:16:49,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:49,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:49,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:49,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:49,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:49,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:49,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741990_1166 (size=12459) 2024-11-10T09:16:49,388 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/6b0fd52a5e8d4e308aaa0835e56f5944 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6b0fd52a5e8d4e308aaa0835e56f5944 2024-11-10T09:16:49,396 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into 6b0fd52a5e8d4e308aaa0835e56f5944(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:49,396 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:49,396 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230209317; duration=0sec 2024-11-10T09:16:49,396 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:49,396 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:16:49,396 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:49,397 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:49,398 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:16:49,398 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:49,398 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b42c3761280742f3b633d691cb11a846, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/831c1aac2169420fa3ff40ee2cf443ae, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/85a8a4340e784451bc051525e8a04ca7] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=35.5 K 2024-11-10T09:16:49,398 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b42c3761280742f3b633d691cb11a846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230206774 2024-11-10T09:16:49,399 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 831c1aac2169420fa3ff40ee2cf443ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731230206949 2024-11-10T09:16:49,399 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a8a4340e784451bc051525e8a04ca7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731230207659 2024-11-10T09:16:49,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741991_1167 (size=4469) 2024-11-10T09:16:49,420 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#142 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:49,421 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/6f432af1e4284918af2235496aec8fcb is 175, key is test_row_0/A:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd8298a750d84b5ca0101150e83a2924_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230208332/Put/seqid=0 2024-11-10T09:16:49,426 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:49,427 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/0b7da559dcbf4f0fa818e60b9949488f is 50, key is test_row_0/C:col10/1731230207659/Put/seqid=0 2024-11-10T09:16:49,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741992_1168 (size=31413) 2024-11-10T09:16:49,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741994_1170 (size=12459) 2024-11-10T09:16:49,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741993_1169 (size=12304) 2024-11-10T09:16:49,453 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/0b7da559dcbf4f0fa818e60b9949488f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/0b7da559dcbf4f0fa818e60b9949488f 2024-11-10T09:16:49,460 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into 0b7da559dcbf4f0fa818e60b9949488f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:49,460 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:49,460 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230209317; duration=0sec 2024-11-10T09:16:49,460 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:49,460 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:16:49,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:49,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230269513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230269514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230269516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230269517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230269627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230269627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230269628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230269628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:16:49,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230269838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230269838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230269838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230269839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:49,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:49,886 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/6f432af1e4284918af2235496aec8fcb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6f432af1e4284918af2235496aec8fcb 2024-11-10T09:16:49,888 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd8298a750d84b5ca0101150e83a2924_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd8298a750d84b5ca0101150e83a2924_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/00f469d3a6684f4c9b6447531f38b2c3, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/00f469d3a6684f4c9b6447531f38b2c3 is 175, key is test_row_0/A:col10/1731230208332/Put/seqid=0 2024-11-10T09:16:49,903 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 6f432af1e4284918af2235496aec8fcb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:49,903 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:49,903 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230209317; duration=0sec 2024-11-10T09:16:49,903 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:49,903 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:16:49,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741995_1171 (size=31105) 2024-11-10T09:16:49,929 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/00f469d3a6684f4c9b6447531f38b2c3 2024-11-10T09:16:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/0444aab0f76047c988585389b3140995 is 50, key is test_row_0/B:col10/1731230208332/Put/seqid=0 2024-11-10T09:16:50,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741996_1172 (size=12151) 2024-11-10T09:16:50,014 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/0444aab0f76047c988585389b3140995 2024-11-10T09:16:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f109959df005451180b3583bf9cef135 is 50, key is test_row_0/C:col10/1731230208332/Put/seqid=0 2024-11-10T09:16:50,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741997_1173 (size=12151) 2024-11-10T09:16:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230270147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230270147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230270157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230270157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,481 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f109959df005451180b3583bf9cef135 2024-11-10T09:16:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/00f469d3a6684f4c9b6447531f38b2c3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3 2024-11-10T09:16:50,530 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3, entries=150, sequenceid=161, filesize=30.4 K 2024-11-10T09:16:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/0444aab0f76047c988585389b3140995 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/0444aab0f76047c988585389b3140995 2024-11-10T09:16:50,544 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/0444aab0f76047c988585389b3140995, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T09:16:50,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f109959df005451180b3583bf9cef135 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f109959df005451180b3583bf9cef135 2024-11-10T09:16:50,553 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f109959df005451180b3583bf9cef135, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T09:16:50,554 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 83bc899d24e43ccf5880b01ce9dff713 in 1176ms, sequenceid=161, compaction requested=false 2024-11-10T09:16:50,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:50,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:50,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T09:16:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T09:16:50,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T09:16:50,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-10T09:16:50,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9630 sec 2024-11-10T09:16:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:50,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:16:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:50,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105475eeef21b848f1a32151124e265639_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230270685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230270687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230270688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230270694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741998_1174 (size=12304) 2024-11-10T09:16:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:16:50,748 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:50,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T09:16:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:50,751 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:50,753 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:50,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:50,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230270798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230270798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230270798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230270817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:50,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:50,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:50,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:50,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230271007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230271007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230271007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230271027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:51,109 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:51,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105475eeef21b848f1a32151124e265639_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105475eeef21b848f1a32151124e265639_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:51,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1c8ccacc98334c7ea2f815f1849f9159, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:51,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1c8ccacc98334c7ea2f815f1849f9159 is 175, key is test_row_0/A:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:51,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741999_1175 (size=31105) 2024-11-10T09:16:51,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230271324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230271324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230271324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230271337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:51,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1c8ccacc98334c7ea2f815f1849f9159 2024-11-10T09:16:51,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/a31dc33f4f3140b0a211cff85bd7a632 is 50, key is test_row_0/B:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:51,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742000_1176 (size=12151) 2024-11-10T09:16:51,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/a31dc33f4f3140b0a211cff85bd7a632 2024-11-10T09:16:51,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/7fdac0a293f3438bb95c83236e44111d is 50, key is test_row_0/C:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:51,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742001_1177 (size=12151) 2024-11-10T09:16:51,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230271841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230271848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230271848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230271858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:51,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:51,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:52,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/7fdac0a293f3438bb95c83236e44111d 2024-11-10T09:16:52,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1c8ccacc98334c7ea2f815f1849f9159 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159 2024-11-10T09:16:52,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159, entries=150, sequenceid=184, filesize=30.4 K 2024-11-10T09:16:52,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/a31dc33f4f3140b0a211cff85bd7a632 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a31dc33f4f3140b0a211cff85bd7a632 2024-11-10T09:16:52,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a31dc33f4f3140b0a211cff85bd7a632, entries=150, sequenceid=184, filesize=11.9 K 2024-11-10T09:16:52,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/7fdac0a293f3438bb95c83236e44111d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/7fdac0a293f3438bb95c83236e44111d 2024-11-10T09:16:52,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/7fdac0a293f3438bb95c83236e44111d, entries=150, sequenceid=184, filesize=11.9 K 2024-11-10T09:16:52,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 83bc899d24e43ccf5880b01ce9dff713 in 1492ms, sequenceid=184, compaction requested=true 2024-11-10T09:16:52,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:52,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:52,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:52,153 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:52,153 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:52,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:52,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:16:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:52,155 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:16:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:52,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:52,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:52,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:52,159 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:52,159 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:52,159 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:52,159 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6b0fd52a5e8d4e308aaa0835e56f5944, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/0444aab0f76047c988585389b3140995, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a31dc33f4f3140b0a211cff85bd7a632] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=35.9 K 2024-11-10T09:16:52,160 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b0fd52a5e8d4e308aaa0835e56f5944, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731230207659 2024-11-10T09:16:52,160 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:52,160 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:52,161 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:52,164 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6f432af1e4284918af2235496aec8fcb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=91.4 K 2024-11-10T09:16:52,164 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:52,164 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6f432af1e4284918af2235496aec8fcb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159] 2024-11-10T09:16:52,167 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f432af1e4284918af2235496aec8fcb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731230207659 2024-11-10T09:16:52,167 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0444aab0f76047c988585389b3140995, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731230208315 2024-11-10T09:16:52,169 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f469d3a6684f4c9b6447531f38b2c3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731230208315 2024-11-10T09:16:52,169 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31dc33f4f3140b0a211cff85bd7a632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731230209499 2024-11-10T09:16:52,171 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8ccacc98334c7ea2f815f1849f9159, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731230209499 2024-11-10T09:16:52,190 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:52,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103226619003754993be239750f04d76d3_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230210692/Put/seqid=0 2024-11-10T09:16:52,191 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:52,199 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110712f97ba6dd34070918d239e4c60cf52_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:52,201 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110712f97ba6dd34070918d239e4c60cf52_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:52,201 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110712f97ba6dd34070918d239e4c60cf52_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:52,203 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/69b53cd37d7f4f8bb5aad3b09c216c5a is 50, key is test_row_0/B:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:52,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742002_1178 (size=12304) 2024-11-10T09:16:52,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742003_1179 (size=4469) 2024-11-10T09:16:52,267 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#152 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:52,268 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/d82450d35099414291d2f7ba89676fe2 is 175, key is test_row_0/A:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:52,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742005_1181 (size=31515) 2024-11-10T09:16:52,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742004_1180 (size=12561) 2024-11-10T09:16:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,651 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103226619003754993be239750f04d76d3_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103226619003754993be239750f04d76d3_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:52,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/56a7dc90dc6f4190ba8351af5e2c9b0b, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:52,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/56a7dc90dc6f4190ba8351af5e2c9b0b is 175, key is test_row_0/A:col10/1731230210692/Put/seqid=0 2024-11-10T09:16:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742006_1182 (size=31105) 2024-11-10T09:16:52,690 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/56a7dc90dc6f4190ba8351af5e2c9b0b 2024-11-10T09:16:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/70c9ac386a034be09d68748bb5cafcd8 is 50, key is test_row_0/B:col10/1731230210692/Put/seqid=0 2024-11-10T09:16:52,726 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/d82450d35099414291d2f7ba89676fe2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/d82450d35099414291d2f7ba89676fe2 2024-11-10T09:16:52,737 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into d82450d35099414291d2f7ba89676fe2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:52,737 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:52,737 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230212153; duration=0sec 2024-11-10T09:16:52,737 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:52,737 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:16:52,737 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:52,744 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/69b53cd37d7f4f8bb5aad3b09c216c5a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/69b53cd37d7f4f8bb5aad3b09c216c5a 2024-11-10T09:16:52,744 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:52,744 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:16:52,744 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:52,745 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/0b7da559dcbf4f0fa818e60b9949488f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f109959df005451180b3583bf9cef135, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/7fdac0a293f3438bb95c83236e44111d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=35.9 K 2024-11-10T09:16:52,746 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7da559dcbf4f0fa818e60b9949488f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731230207659 2024-11-10T09:16:52,747 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f109959df005451180b3583bf9cef135, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731230208315 2024-11-10T09:16:52,747 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fdac0a293f3438bb95c83236e44111d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731230209499 2024-11-10T09:16:52,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742007_1183 (size=12151) 2024-11-10T09:16:52,753 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into 69b53cd37d7f4f8bb5aad3b09c216c5a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:52,753 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:52,753 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230212153; duration=0sec 2024-11-10T09:16:52,753 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:52,753 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:16:52,754 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/70c9ac386a034be09d68748bb5cafcd8 2024-11-10T09:16:52,773 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:52,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/732d8ba7d35f4b8b942e84c1add4e618 is 50, key is test_row_0/C:col10/1731230210692/Put/seqid=0 2024-11-10T09:16:52,774 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f73c588ffd6f4210b22456d2ffb74dbb is 50, key is test_row_0/C:col10/1731230209513/Put/seqid=0 2024-11-10T09:16:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:52,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742008_1184 (size=12151) 2024-11-10T09:16:52,832 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/732d8ba7d35f4b8b942e84c1add4e618 2024-11-10T09:16:52,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/56a7dc90dc6f4190ba8351af5e2c9b0b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b 2024-11-10T09:16:52,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742009_1185 (size=12561) 2024-11-10T09:16:52,856 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b, entries=150, sequenceid=196, filesize=30.4 K 2024-11-10T09:16:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/70c9ac386a034be09d68748bb5cafcd8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/70c9ac386a034be09d68748bb5cafcd8 2024-11-10T09:16:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,863 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f73c588ffd6f4210b22456d2ffb74dbb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f73c588ffd6f4210b22456d2ffb74dbb 2024-11-10T09:16:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,868 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/70c9ac386a034be09d68748bb5cafcd8, entries=150, sequenceid=196, filesize=11.9 K 2024-11-10T09:16:52,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/732d8ba7d35f4b8b942e84c1add4e618 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/732d8ba7d35f4b8b942e84c1add4e618 2024-11-10T09:16:52,875 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into f73c588ffd6f4210b22456d2ffb74dbb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:52,875 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:52,875 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230212156; duration=0sec 2024-11-10T09:16:52,876 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:52,876 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:52,899 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/732d8ba7d35f4b8b942e84c1add4e618, entries=150, sequenceid=196, filesize=11.9 K 2024-11-10T09:16:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,909 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=67.09 KB/68700 for 83bc899d24e43ccf5880b01ce9dff713 in 753ms, sequenceid=196, compaction requested=false 2024-11-10T09:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T09:16:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T09:16:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:16:52,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:52,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:52,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:52,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:52,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:52,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T09:16:52,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1580 sec 2024-11-10T09:16:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1740 sec 2024-11-10T09:16:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102564ab2feca540c2b545694173a1b376_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742011_1187 (size=27248) 2024-11-10T09:16:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102564ab2feca540c2b545694173a1b376_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102564ab2feca540c2b545694173a1b376_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/a418b08908d84e289ea783d48a9ec46f, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:52,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/a418b08908d84e289ea783d48a9ec46f is 175, key is test_row_0/A:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742010_1186 (size=83035) 2024-11-10T09:16:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230273023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230273024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230273032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230273035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230273040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230273147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230273147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,161 INFO [master/802346a334f5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T09:16:53,162 INFO [master/802346a334f5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T09:16:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230273158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230273158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230273158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230273367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230273368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230273388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/a418b08908d84e289ea783d48a9ec46f 2024-11-10T09:16:53,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/b907ea51d1b740f8a6e3a54327e29ed8 is 50, key is test_row_0/B:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742012_1188 (size=12151) 2024-11-10T09:16:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230273688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230273688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230273698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230273698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230273701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:53,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/b907ea51d1b740f8a6e3a54327e29ed8 2024-11-10T09:16:53,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/287c7983114b49c7beaa6d3ed5ff3f75 is 50, key is test_row_0/C:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:53,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742013_1189 (size=12151) 2024-11-10T09:16:53,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/287c7983114b49c7beaa6d3ed5ff3f75 2024-11-10T09:16:53,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/a418b08908d84e289ea783d48a9ec46f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f 2024-11-10T09:16:53,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f, entries=450, sequenceid=213, filesize=81.1 K 2024-11-10T09:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/b907ea51d1b740f8a6e3a54327e29ed8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/b907ea51d1b740f8a6e3a54327e29ed8 2024-11-10T09:16:53,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/b907ea51d1b740f8a6e3a54327e29ed8, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T09:16:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/287c7983114b49c7beaa6d3ed5ff3f75 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/287c7983114b49c7beaa6d3ed5ff3f75 2024-11-10T09:16:53,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/287c7983114b49c7beaa6d3ed5ff3f75, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T09:16:53,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 83bc899d24e43ccf5880b01ce9dff713 in 1028ms, sequenceid=213, compaction requested=true 2024-11-10T09:16:53,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:53,942 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:53,945 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 145655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:53,945 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:53,945 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:53,945 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/d82450d35099414291d2f7ba89676fe2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=142.2 K 2024-11-10T09:16:53,947 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:53,947 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/d82450d35099414291d2f7ba89676fe2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f] 2024-11-10T09:16:53,951 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d82450d35099414291d2f7ba89676fe2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731230209499 2024-11-10T09:16:53,952 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a7dc90dc6f4190ba8351af5e2c9b0b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731230210685 2024-11-10T09:16:53,953 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a418b08908d84e289ea783d48a9ec46f, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731230212829 2024-11-10T09:16:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:53,969 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:53,970 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:53,971 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:53,971 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:53,972 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:53,972 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/69b53cd37d7f4f8bb5aad3b09c216c5a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/70c9ac386a034be09d68748bb5cafcd8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/b907ea51d1b740f8a6e3a54327e29ed8] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=36.0 K 2024-11-10T09:16:53,981 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b53cd37d7f4f8bb5aad3b09c216c5a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731230209499 2024-11-10T09:16:53,982 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c9ac386a034be09d68748bb5cafcd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731230210685 2024-11-10T09:16:53,982 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b907ea51d1b740f8a6e3a54327e29ed8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731230212849 2024-11-10T09:16:53,990 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101fc4254eb32a4156800d5fd2a8b726c1_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:53,994 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101fc4254eb32a4156800d5fd2a8b726c1_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:53,994 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101fc4254eb32a4156800d5fd2a8b726c1_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:54,015 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:54,016 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/38ce3af92f7743f38ca8ff9ebcf230bc is 50, key is test_row_0/B:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742014_1190 (size=4469) 2024-11-10T09:16:54,102 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#159 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:54,103 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/243d3f6ccdd241119c1f9a2bbf854518 is 175, key is test_row_0/A:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:54,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742015_1191 (size=12663) 2024-11-10T09:16:54,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742016_1192 (size=31617) 2024-11-10T09:16:54,194 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/243d3f6ccdd241119c1f9a2bbf854518 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/243d3f6ccdd241119c1f9a2bbf854518 2024-11-10T09:16:54,202 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 243d3f6ccdd241119c1f9a2bbf854518(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:54,202 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:54,202 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230213942; duration=0sec 2024-11-10T09:16:54,202 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:54,202 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:16:54,203 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:54,204 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:54,204 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:16:54,204 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:54,204 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f73c588ffd6f4210b22456d2ffb74dbb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/732d8ba7d35f4b8b942e84c1add4e618, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/287c7983114b49c7beaa6d3ed5ff3f75] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=36.0 K 2024-11-10T09:16:54,205 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f73c588ffd6f4210b22456d2ffb74dbb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731230209499 2024-11-10T09:16:54,206 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732d8ba7d35f4b8b942e84c1add4e618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731230210685 2024-11-10T09:16:54,206 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287c7983114b49c7beaa6d3ed5ff3f75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731230212849 2024-11-10T09:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:54,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:16:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:54,231 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:54,232 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/56048900ec084d3a8305f932ce5d0fa5 is 50, key is test_row_0/C:col10/1731230212849/Put/seqid=0 2024-11-10T09:16:54,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032bb03ec49ea4479b844ecae07a24521_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230213031/Put/seqid=0 2024-11-10T09:16:54,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230274244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230274245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230274247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230274253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742018_1194 (size=14794) 2024-11-10T09:16:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230274265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742017_1193 (size=12663) 2024-11-10T09:16:54,310 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/56048900ec084d3a8305f932ce5d0fa5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/56048900ec084d3a8305f932ce5d0fa5 2024-11-10T09:16:54,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032bb03ec49ea4479b844ecae07a24521_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032bb03ec49ea4479b844ecae07a24521_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:54,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/0e47f92a69284fe0b480faa23de66d33, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:54,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/0e47f92a69284fe0b480faa23de66d33 is 175, key is test_row_0/A:col10/1731230213031/Put/seqid=0 2024-11-10T09:16:54,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into 56048900ec084d3a8305f932ce5d0fa5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:54,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:54,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230213969; duration=0sec 2024-11-10T09:16:54,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:54,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:16:54,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742019_1195 (size=39749) 2024-11-10T09:16:54,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230274377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230274377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230274377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230274377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230274387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,541 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/38ce3af92f7743f38ca8ff9ebcf230bc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/38ce3af92f7743f38ca8ff9ebcf230bc 2024-11-10T09:16:54,550 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into 38ce3af92f7743f38ca8ff9ebcf230bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:54,550 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:54,550 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230213969; duration=0sec 2024-11-10T09:16:54,550 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:54,550 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:16:54,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230274587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230274597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/0e47f92a69284fe0b480faa23de66d33 2024-11-10T09:16:54,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/a3dad0dbc8954fc39dced6e8903bf404 is 50, key is test_row_0/B:col10/1731230213031/Put/seqid=0 2024-11-10T09:16:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742020_1196 (size=12151) 2024-11-10T09:16:54,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230274898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230274898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230274898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230274898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230274908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:54,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:16:54,959 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:54,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T09:16:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:16:54,963 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:54,964 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:54,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:16:55,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/a3dad0dbc8954fc39dced6e8903bf404 2024-11-10T09:16:55,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:16:55,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:55,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:16:55,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/3c6307650253407388b5652a6a4a5335 is 50, key is test_row_0/C:col10/1731230213031/Put/seqid=0 2024-11-10T09:16:55,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742021_1197 (size=12151) 2024-11-10T09:16:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230275417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230275417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230275417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:55,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:16:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230275467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230275468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:55,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:16:55,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:55,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:16:55,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/3c6307650253407388b5652a6a4a5335 2024-11-10T09:16:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/0e47f92a69284fe0b480faa23de66d33 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33 2024-11-10T09:16:55,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33, entries=200, sequenceid=234, filesize=38.8 K 2024-11-10T09:16:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/a3dad0dbc8954fc39dced6e8903bf404 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a3dad0dbc8954fc39dced6e8903bf404 2024-11-10T09:16:55,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a3dad0dbc8954fc39dced6e8903bf404, entries=150, sequenceid=234, filesize=11.9 K 2024-11-10T09:16:55,713 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T09:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/3c6307650253407388b5652a6a4a5335 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/3c6307650253407388b5652a6a4a5335 2024-11-10T09:16:55,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/3c6307650253407388b5652a6a4a5335, entries=150, sequenceid=234, filesize=11.9 K 2024-11-10T09:16:55,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 83bc899d24e43ccf5880b01ce9dff713 in 1502ms, sequenceid=234, compaction requested=false 2024-11-10T09:16:55,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:55,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:16:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:55,735 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:16:55,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:55,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:55,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:55,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:55,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:55,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110da1e26c761f849bfa455d264e58cec7f_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230214239/Put/seqid=0 2024-11-10T09:16:55,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742022_1198 (size=12304) 2024-11-10T09:16:55,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:55,784 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110da1e26c761f849bfa455d264e58cec7f_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110da1e26c761f849bfa455d264e58cec7f_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/8f3e3cc23ba4444987c8cf2e55aaa959, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/8f3e3cc23ba4444987c8cf2e55aaa959 is 175, key is test_row_0/A:col10/1731230214239/Put/seqid=0 2024-11-10T09:16:55,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742023_1199 (size=31105) 2024-11-10T09:16:55,810 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/8f3e3cc23ba4444987c8cf2e55aaa959 2024-11-10T09:16:55,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/ef1803cac415447485ae698edc768f39 is 50, key is test_row_0/B:col10/1731230214239/Put/seqid=0 2024-11-10T09:16:55,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742024_1200 (size=12151) 2024-11-10T09:16:55,848 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/ef1803cac415447485ae698edc768f39 2024-11-10T09:16:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/1dfd70c65d8e49da97923ac62ce41933 is 50, key is test_row_0/C:col10/1731230214239/Put/seqid=0 2024-11-10T09:16:55,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742025_1201 (size=12151) 2024-11-10T09:16:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:16:56,269 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/1dfd70c65d8e49da97923ac62ce41933 2024-11-10T09:16:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/8f3e3cc23ba4444987c8cf2e55aaa959 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959 2024-11-10T09:16:56,281 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959, entries=150, sequenceid=248, filesize=30.4 K 2024-11-10T09:16:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/ef1803cac415447485ae698edc768f39 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ef1803cac415447485ae698edc768f39 2024-11-10T09:16:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,287 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ef1803cac415447485ae698edc768f39, entries=150, sequenceid=248, filesize=11.9 K 2024-11-10T09:16:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/1dfd70c65d8e49da97923ac62ce41933 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1dfd70c65d8e49da97923ac62ce41933 2024-11-10T09:16:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,294 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1dfd70c65d8e49da97923ac62ce41933, entries=150, sequenceid=248, filesize=11.9 K 2024-11-10T09:16:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,294 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 83bc899d24e43ccf5880b01ce9dff713 in 559ms, sequenceid=248, compaction requested=true 2024-11-10T09:16:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T09:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T09:16:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T09:16:56,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-10T09:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.3390 sec 2024-11-10T09:16:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:56,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:16:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e915a7a7b8c94f9fb6e3675ab3686051_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731230216491, max timestamp=1731230216537Num uniques 859; 2024-11-10T09:16:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731230216491, max timestamp=1731230216537Num uniques 859; 2024-11-10T09:16:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T09:16:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T09:16:56,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731230216491, max timestamp=1731230216537Num uniques 859; 2024-11-10T09:16:56,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T09:16:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230276541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742027_1203 (size=24908) 2024-11-10T09:16:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,560 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230276547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230276549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230276550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e915a7a7b8c94f9fb6e3675ab3686051_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e915a7a7b8c94f9fb6e3675ab3686051_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/352ea9e8b26547b3854992838f50f451, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230276545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/352ea9e8b26547b3854992838f50f451 is 175, key is test_row_0/A:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742026_1202 (size=74545) 2024-11-10T09:16:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230276667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230276678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230276678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230276679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230276679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230276887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230276887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230276887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230276887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230276897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:56,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/352ea9e8b26547b3854992838f50f451 2024-11-10T09:16:56,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/781ac33603924193a2c855208e1db128 is 50, key is test_row_0/B:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742028_1204 (size=12251) 2024-11-10T09:16:57,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/781ac33603924193a2c855208e1db128 2024-11-10T09:16:57,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/1c23f7000df640b599bb0b47a6b9eb84 is 50, key is test_row_0/C:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:57,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742029_1205 (size=12251) 2024-11-10T09:16:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:16:57,108 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:57,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T09:16:57,114 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:16:57,114 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:57,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230277197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230277198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230277199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230277199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:16:57,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230277219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:16:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:57,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:16:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:16:57,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/1c23f7000df640b599bb0b47a6b9eb84 2024-11-10T09:16:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/352ea9e8b26547b3854992838f50f451 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451 2024-11-10T09:16:57,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451, entries=400, sequenceid=259, filesize=72.8 K 2024-11-10T09:16:57,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/781ac33603924193a2c855208e1db128 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/781ac33603924193a2c855208e1db128 2024-11-10T09:16:57,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/781ac33603924193a2c855208e1db128, entries=150, sequenceid=259, filesize=12.0 K 2024-11-10T09:16:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/1c23f7000df640b599bb0b47a6b9eb84 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1c23f7000df640b599bb0b47a6b9eb84 2024-11-10T09:16:57,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1c23f7000df640b599bb0b47a6b9eb84, entries=150, sequenceid=259, filesize=12.0 K 2024-11-10T09:16:57,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=147.60 KB/151140 for 83bc899d24e43ccf5880b01ce9dff713 in 1035ms, sequenceid=259, compaction requested=true 2024-11-10T09:16:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:57,525 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:57,525 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:57,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:57,528 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 177016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:57,528 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:57,529 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,529 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/243d3f6ccdd241119c1f9a2bbf854518, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=172.9 K 2024-11-10T09:16:57,529 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,529 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/243d3f6ccdd241119c1f9a2bbf854518, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451] 2024-11-10T09:16:57,529 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:57,529 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:57,529 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,529 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/38ce3af92f7743f38ca8ff9ebcf230bc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a3dad0dbc8954fc39dced6e8903bf404, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ef1803cac415447485ae698edc768f39, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/781ac33603924193a2c855208e1db128] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=48.1 K 2024-11-10T09:16:57,529 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243d3f6ccdd241119c1f9a2bbf854518, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731230212849 2024-11-10T09:16:57,530 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ce3af92f7743f38ca8ff9ebcf230bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731230212849 2024-11-10T09:16:57,530 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e47f92a69284fe0b480faa23de66d33, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731230213007 2024-11-10T09:16:57,530 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting a3dad0dbc8954fc39dced6e8903bf404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731230213007 2024-11-10T09:16:57,530 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f3e3cc23ba4444987c8cf2e55aaa959, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731230214239 2024-11-10T09:16:57,531 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 352ea9e8b26547b3854992838f50f451, keycount=400, bloomtype=ROW, size=72.8 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731230216465 2024-11-10T09:16:57,531 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1803cac415447485ae698edc768f39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731230214239 2024-11-10T09:16:57,531 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 781ac33603924193a2c855208e1db128, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731230216467 2024-11-10T09:16:57,545 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:57,547 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110a8140ec0cb4b442fbf8b7ef42bd2b1d6_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:57,548 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#172 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:57,549 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/bb0cfbf6e81b4f368888897e905a0b1b is 50, key is test_row_0/B:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:57,551 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110a8140ec0cb4b442fbf8b7ef42bd2b1d6_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:57,551 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a8140ec0cb4b442fbf8b7ef42bd2b1d6_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:57,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742030_1206 (size=12899) 2024-11-10T09:16:57,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:16:57,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,575 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=147.60 KB heapSize=359.20 KB 2024-11-10T09:16:57,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:16:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742031_1207 (size=4469) 2024-11-10T09:16:57,579 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#171 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:57,580 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/4f7ed136d18c43f6bae080fd1273d729 is 175, key is test_row_0/A:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c18d70b44f344a7eae4b64e633165463_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230216548/Put/seqid=0 2024-11-10T09:16:57,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742032_1208 (size=31853) 2024-11-10T09:16:57,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742033_1209 (size=12454) 2024-11-10T09:16:57,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:57,701 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c18d70b44f344a7eae4b64e633165463_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c18d70b44f344a7eae4b64e633165463_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:57,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1720c2537f80407f8106084629e5a950, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1720c2537f80407f8106084629e5a950 is 175, key is test_row_0/A:col10/1731230216548/Put/seqid=0 2024-11-10T09:16:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:16:57,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742034_1210 (size=31255) 2024-11-10T09:16:57,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230277726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230277726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230277727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230277741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230277744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230277847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230277847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230277857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230277857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:57,975 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/bb0cfbf6e81b4f368888897e905a0b1b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/bb0cfbf6e81b4f368888897e905a0b1b 2024-11-10T09:16:57,981 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into bb0cfbf6e81b4f368888897e905a0b1b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:57,981 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:57,981 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=12, startTime=1731230217525; duration=0sec 2024-11-10T09:16:57,981 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:57,981 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:16:57,981 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:16:57,982 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:16:57,982 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:16:57,982 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:57,983 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/56048900ec084d3a8305f932ce5d0fa5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/3c6307650253407388b5652a6a4a5335, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1dfd70c65d8e49da97923ac62ce41933, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1c23f7000df640b599bb0b47a6b9eb84] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=48.1 K 2024-11-10T09:16:57,983 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 56048900ec084d3a8305f932ce5d0fa5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731230212849 2024-11-10T09:16:57,983 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6307650253407388b5652a6a4a5335, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731230213007 2024-11-10T09:16:57,983 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfd70c65d8e49da97923ac62ce41933, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731230214239 2024-11-10T09:16:57,984 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c23f7000df640b599bb0b47a6b9eb84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731230216467 2024-11-10T09:16:57,995 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:57,995 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f8750197c20a4df1817f78edaabac43d is 50, key is test_row_0/C:col10/1731230216467/Put/seqid=0 2024-11-10T09:16:58,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742035_1211 (size=12899) 2024-11-10T09:16:58,047 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/4f7ed136d18c43f6bae080fd1273d729 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4f7ed136d18c43f6bae080fd1273d729 2024-11-10T09:16:58,052 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 4f7ed136d18c43f6bae080fd1273d729(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:58,052 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:58,052 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=12, startTime=1731230217525; duration=0sec 2024-11-10T09:16:58,052 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:58,052 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:16:58,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230278057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230278058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230278077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230278077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,142 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1720c2537f80407f8106084629e5a950 2024-11-10T09:16:58,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5790d3da37d647e48a6d5688e4f19dbf is 50, key is test_row_0/B:col10/1731230216548/Put/seqid=0 2024-11-10T09:16:58,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742036_1212 (size=12301) 2024-11-10T09:16:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:16:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230278367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230278367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230278387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230278387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,420 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/f8750197c20a4df1817f78edaabac43d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f8750197c20a4df1817f78edaabac43d 2024-11-10T09:16:58,429 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into f8750197c20a4df1817f78edaabac43d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:16:58,429 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:58,429 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=12, startTime=1731230217525; duration=0sec 2024-11-10T09:16:58,429 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:58,430 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:16:58,553 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5790d3da37d647e48a6d5688e4f19dbf 2024-11-10T09:16:58,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/facb4e2bb0bd43a2befb102e9aa9567b is 50, key is test_row_0/C:col10/1731230216548/Put/seqid=0 2024-11-10T09:16:58,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742037_1213 (size=12301) 2024-11-10T09:16:58,565 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/facb4e2bb0bd43a2befb102e9aa9567b 2024-11-10T09:16:58,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/1720c2537f80407f8106084629e5a950 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950 2024-11-10T09:16:58,575 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950, entries=150, sequenceid=284, filesize=30.5 K 2024-11-10T09:16:58,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5790d3da37d647e48a6d5688e4f19dbf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5790d3da37d647e48a6d5688e4f19dbf 2024-11-10T09:16:58,581 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5790d3da37d647e48a6d5688e4f19dbf, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T09:16:58,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/facb4e2bb0bd43a2befb102e9aa9567b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/facb4e2bb0bd43a2befb102e9aa9567b 2024-11-10T09:16:58,587 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/facb4e2bb0bd43a2befb102e9aa9567b, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T09:16:58,587 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~359.16 KB/367776, currentSize=53.67 KB/54960 for 83bc899d24e43ccf5880b01ce9dff713 in 1012ms, sequenceid=284, compaction requested=false 2024-11-10T09:16:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T09:16:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T09:16:58,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T09:16:58,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-10T09:16:58,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4820 sec 2024-11-10T09:16:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:58,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:16:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:58,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104747b5795f5741e58f12a7c4f57b398e_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230218760/Put/seqid=0 2024-11-10T09:16:58,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742038_1214 (size=14994) 2024-11-10T09:16:58,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230278879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230278887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230278888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230278897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230278899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:58,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230278987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230279197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:59,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104747b5795f5741e58f12a7c4f57b398e_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104747b5795f5741e58f12a7c4f57b398e_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:59,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/6fe320c1b49f404abe00c2c0a324bbf7, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:59,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/6fe320c1b49f404abe00c2c0a324bbf7 is 175, key is test_row_0/A:col10/1731230218760/Put/seqid=0 2024-11-10T09:16:59,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742039_1215 (size=39949) 2024-11-10T09:16:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:16:59,257 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:16:59,258 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:16:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T09:16:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:16:59,262 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:16:59,262 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:16:59,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:16:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:16:59,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:16:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230279507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:16:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:16:59,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/6fe320c1b49f404abe00c2c0a324bbf7 2024-11-10T09:16:59,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/6408a83e8fca408aa4340133708e3e44 is 50, key is test_row_0/B:col10/1731230218760/Put/seqid=0 2024-11-10T09:16:59,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742040_1216 (size=12301) 2024-11-10T09:16:59,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/6408a83e8fca408aa4340133708e3e44 2024-11-10T09:16:59,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/090f1c632ab6404a9f3482cc42e6864f is 50, key is test_row_0/C:col10/1731230218760/Put/seqid=0 2024-11-10T09:16:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742041_1217 (size=12301) 2024-11-10T09:16:59,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/090f1c632ab6404a9f3482cc42e6864f 2024-11-10T09:16:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/6fe320c1b49f404abe00c2c0a324bbf7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7 2024-11-10T09:16:59,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7, entries=200, sequenceid=299, filesize=39.0 K 2024-11-10T09:16:59,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/6408a83e8fca408aa4340133708e3e44 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6408a83e8fca408aa4340133708e3e44 2024-11-10T09:16:59,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6408a83e8fca408aa4340133708e3e44, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T09:16:59,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/090f1c632ab6404a9f3482cc42e6864f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/090f1c632ab6404a9f3482cc42e6864f 2024-11-10T09:16:59,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:16:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:16:59,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/090f1c632ab6404a9f3482cc42e6864f, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T09:16:59,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 83bc899d24e43ccf5880b01ce9dff713 in 972ms, sequenceid=299, compaction requested=true 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:16:59,733 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:59,733 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:16:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:16:59,734 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:59,734 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:16:59,734 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:16:59,734 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:16:59,734 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,734 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,734 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/bb0cfbf6e81b4f368888897e905a0b1b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5790d3da37d647e48a6d5688e4f19dbf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6408a83e8fca408aa4340133708e3e44] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=36.6 K 2024-11-10T09:16:59,734 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4f7ed136d18c43f6bae080fd1273d729, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=100.6 K 2024-11-10T09:16:59,734 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,734 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4f7ed136d18c43f6bae080fd1273d729, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7] 2024-11-10T09:16:59,735 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0cfbf6e81b4f368888897e905a0b1b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731230216467 2024-11-10T09:16:59,735 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f7ed136d18c43f6bae080fd1273d729, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731230216467 2024-11-10T09:16:59,735 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5790d3da37d647e48a6d5688e4f19dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731230216537 2024-11-10T09:16:59,735 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1720c2537f80407f8106084629e5a950, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731230216537 2024-11-10T09:16:59,735 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6408a83e8fca408aa4340133708e3e44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731230217739 2024-11-10T09:16:59,735 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fe320c1b49f404abe00c2c0a324bbf7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731230217720 2024-11-10T09:16:59,741 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:59,742 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:16:59,742 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/4b88de2242e54b1ba534317655f17e0d is 50, key is test_row_0/B:col10/1731230218760/Put/seqid=0 2024-11-10T09:16:59,744 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411109182d0a720174026a0d0f329d8707486_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:59,747 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411109182d0a720174026a0d0f329d8707486_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:59,747 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109182d0a720174026a0d0f329d8707486_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:59,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742042_1218 (size=13051) 2024-11-10T09:16:59,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742043_1219 (size=4469) 2024-11-10T09:16:59,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:16:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:16:59,880 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:16:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:16:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:16:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:16:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:16:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101bfd754956454b3e911818bc8bc15e1e_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230218855/Put/seqid=0 2024-11-10T09:16:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:16:59,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742044_1220 (size=12454) 2024-11-10T09:16:59,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:16:59,898 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101bfd754956454b3e911818bc8bc15e1e_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101bfd754956454b3e911818bc8bc15e1e_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:59,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/84447571451f4dac8002177be8e63d89, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:16:59,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/84447571451f4dac8002177be8e63d89 is 175, key is test_row_0/A:col10/1731230218855/Put/seqid=0 2024-11-10T09:16:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:16:59,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:16:59,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230279910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742045_1221 (size=31255) 2024-11-10T09:16:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230279912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,920 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/84447571451f4dac8002177be8e63d89 2024-11-10T09:16:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230279917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:59,925 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 83bc899d24e43ccf5880b01ce9dff713 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:16:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:16:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230279919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:16:59,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/3c4800d5a6ef4adda8b33161b1e030c5 is 50, key is test_row_0/B:col10/1731230218855/Put/seqid=0 2024-11-10T09:16:59,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742046_1222 (size=12301) 2024-11-10T09:17:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230280027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230280027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230280027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230280037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,162 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:00,163 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/65c31f411a544b1a896a91b08c0534c7 is 175, key is test_row_0/A:col10/1731230218760/Put/seqid=0 2024-11-10T09:17:00,165 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/4b88de2242e54b1ba534317655f17e0d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4b88de2242e54b1ba534317655f17e0d 2024-11-10T09:17:00,172 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into 4b88de2242e54b1ba534317655f17e0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:00,172 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:00,172 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230219733; duration=0sec 2024-11-10T09:17:00,173 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:00,173 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:17:00,173 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:00,174 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:00,174 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:17:00,174 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:00,174 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f8750197c20a4df1817f78edaabac43d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/facb4e2bb0bd43a2befb102e9aa9567b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/090f1c632ab6404a9f3482cc42e6864f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=36.6 K 2024-11-10T09:17:00,175 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f8750197c20a4df1817f78edaabac43d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731230216467 2024-11-10T09:17:00,175 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting facb4e2bb0bd43a2befb102e9aa9567b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731230216537 2024-11-10T09:17:00,175 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 090f1c632ab6404a9f3482cc42e6864f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731230217739 2024-11-10T09:17:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742047_1223 (size=32005) 2024-11-10T09:17:00,187 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:00,187 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/9de752660a5c47cf930a1c191829d89c is 50, key is test_row_0/C:col10/1731230218760/Put/seqid=0 2024-11-10T09:17:00,204 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/65c31f411a544b1a896a91b08c0534c7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/65c31f411a544b1a896a91b08c0534c7 2024-11-10T09:17:00,215 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 65c31f411a544b1a896a91b08c0534c7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:00,215 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:00,215 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230219733; duration=0sec 2024-11-10T09:17:00,215 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:00,215 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:17:00,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742048_1224 (size=13051) 2024-11-10T09:17:00,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230280238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230280238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,240 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/9de752660a5c47cf930a1c191829d89c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/9de752660a5c47cf930a1c191829d89c 2024-11-10T09:17:00,245 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into 9de752660a5c47cf930a1c191829d89c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:00,245 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:00,245 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230219733; duration=0sec 2024-11-10T09:17:00,246 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:00,246 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:17:00,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230280247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,348 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/3c4800d5a6ef4adda8b33161b1e030c5 2024-11-10T09:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/8455797446fc40b19a51211051200a35 is 50, key is test_row_0/C:col10/1731230218855/Put/seqid=0 2024-11-10T09:17:00,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742049_1225 (size=12301) 2024-11-10T09:17:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:17:00,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230280547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230280547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230280557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:00,762 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/8455797446fc40b19a51211051200a35 2024-11-10T09:17:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/84447571451f4dac8002177be8e63d89 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89 2024-11-10T09:17:00,771 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89, entries=150, sequenceid=319, filesize=30.5 K 2024-11-10T09:17:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/3c4800d5a6ef4adda8b33161b1e030c5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/3c4800d5a6ef4adda8b33161b1e030c5 2024-11-10T09:17:00,776 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/3c4800d5a6ef4adda8b33161b1e030c5, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T09:17:00,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/8455797446fc40b19a51211051200a35 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/8455797446fc40b19a51211051200a35 2024-11-10T09:17:00,786 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/8455797446fc40b19a51211051200a35, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T09:17:00,787 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 83bc899d24e43ccf5880b01ce9dff713 in 907ms, sequenceid=319, compaction requested=false 2024-11-10T09:17:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T09:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T09:17:00,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T09:17:00,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-11-10T09:17:00,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5320 sec 2024-11-10T09:17:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:01,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:17:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:17:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:17:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:17:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:01,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ef9326d685d34dde994b8379c346943b_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742050_1226 (size=14994) 2024-11-10T09:17:01,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:01,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ef9326d685d34dde994b8379c346943b_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ef9326d685d34dde994b8379c346943b_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:01,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/222a1488792d41e2b3a6270099a61d8e, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:01,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/222a1488792d41e2b3a6270099a61d8e is 175, key is test_row_0/A:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742051_1227 (size=39949) 2024-11-10T09:17:01,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/222a1488792d41e2b3a6270099a61d8e 2024-11-10T09:17:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230281089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230281089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230281091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230281092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/954347ea1c324aab90a25ad412816fd3 is 50, key is test_row_0/B:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742052_1228 (size=12301) 2024-11-10T09:17:01,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/954347ea1c324aab90a25ad412816fd3 2024-11-10T09:17:01,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/b48918c597e94dce83b2315a1ba0f58a is 50, key is test_row_0/C:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742053_1229 (size=12301) 2024-11-10T09:17:01,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/b48918c597e94dce83b2315a1ba0f58a 2024-11-10T09:17:01,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/222a1488792d41e2b3a6270099a61d8e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e 2024-11-10T09:17:01,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e, entries=200, sequenceid=335, filesize=39.0 K 2024-11-10T09:17:01,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/954347ea1c324aab90a25ad412816fd3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/954347ea1c324aab90a25ad412816fd3 2024-11-10T09:17:01,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/954347ea1c324aab90a25ad412816fd3, entries=150, sequenceid=335, filesize=12.0 K 2024-11-10T09:17:01,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/b48918c597e94dce83b2315a1ba0f58a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b48918c597e94dce83b2315a1ba0f58a 2024-11-10T09:17:01,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b48918c597e94dce83b2315a1ba0f58a, entries=150, sequenceid=335, filesize=12.0 K 2024-11-10T09:17:01,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 83bc899d24e43ccf5880b01ce9dff713 in 144ms, sequenceid=335, compaction requested=true 2024-11-10T09:17:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:01,185 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 83bc899d24e43ccf5880b01ce9dff713:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:17:01,185 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:01,186 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:01,186 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:01,187 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/A is initiating minor compaction (all files) 2024-11-10T09:17:01,187 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/B is initiating minor compaction (all files) 2024-11-10T09:17:01,187 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/B in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,187 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4b88de2242e54b1ba534317655f17e0d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/3c4800d5a6ef4adda8b33161b1e030c5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/954347ea1c324aab90a25ad412816fd3] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=36.8 K 2024-11-10T09:17:01,187 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/A in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,187 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/65c31f411a544b1a896a91b08c0534c7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=100.8 K 2024-11-10T09:17:01,187 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,187 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/65c31f411a544b1a896a91b08c0534c7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e] 2024-11-10T09:17:01,187 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b88de2242e54b1ba534317655f17e0d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731230217739 2024-11-10T09:17:01,187 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c31f411a544b1a896a91b08c0534c7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731230217739 2024-11-10T09:17:01,188 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4800d5a6ef4adda8b33161b1e030c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230218837 2024-11-10T09:17:01,188 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84447571451f4dac8002177be8e63d89, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230218837 2024-11-10T09:17:01,188 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 954347ea1c324aab90a25ad412816fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731230219910 2024-11-10T09:17:01,188 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222a1488792d41e2b3a6270099a61d8e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731230219909 2024-11-10T09:17:01,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-10T09:17:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:01,210 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:01,211 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/59f2a30f8e3e44b591a9c1284841ea56 is 50, key is test_row_0/B:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230221085, max timestamp=1731230221207Num uniques 859; 2024-11-10T09:17:01,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:01,212 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:17:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:17:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:17:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:01,231 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111078373710760f4aa9868af492a82674b1_83bc899d24e43ccf5880b01ce9dff713 store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:01,234 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111078373710760f4aa9868af492a82674b1_83bc899d24e43ccf5880b01ce9dff713, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:01,234 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111078373710760f4aa9868af492a82674b1_83bc899d24e43ccf5880b01ce9dff713 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:01,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742054_1230 (size=13153) 2024-11-10T09:17:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230281269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230281270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230281272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230281273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c41e412cd0774b848693d51d4fc97473_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230221207/Put/seqid=0 2024-11-10T09:17:01,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742055_1231 (size=4469) 2024-11-10T09:17:01,317 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#A#compaction#190 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:01,317 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/48d48a68666e4ab99a4c9daa47b5f801 is 175, key is test_row_0/A:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742057_1233 (size=32107) 2024-11-10T09:17:01,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742056_1232 (size=12454) 2024-11-10T09:17:01,367 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/48d48a68666e4ab99a4c9daa47b5f801 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/48d48a68666e4ab99a4c9daa47b5f801 2024-11-10T09:17:01,375 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/A of 83bc899d24e43ccf5880b01ce9dff713 into 48d48a68666e4ab99a4c9daa47b5f801(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:01,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:01,375 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/A, priority=13, startTime=1731230221184; duration=0sec 2024-11-10T09:17:01,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:01,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:A 2024-11-10T09:17:01,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:01,377 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:01,378 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 83bc899d24e43ccf5880b01ce9dff713/C is initiating minor compaction (all files) 2024-11-10T09:17:01,378 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 83bc899d24e43ccf5880b01ce9dff713/C in TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,378 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/9de752660a5c47cf930a1c191829d89c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/8455797446fc40b19a51211051200a35, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b48918c597e94dce83b2315a1ba0f58a] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp, totalSize=36.8 K 2024-11-10T09:17:01,378 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de752660a5c47cf930a1c191829d89c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731230217739 2024-11-10T09:17:01,379 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8455797446fc40b19a51211051200a35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230218837 2024-11-10T09:17:01,380 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48918c597e94dce83b2315a1ba0f58a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731230219910 2024-11-10T09:17:01,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230281387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230281387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230281387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230281387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,398 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 83bc899d24e43ccf5880b01ce9dff713#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:01,399 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/c1e01ef43ebb423e82380141350a8f43 is 50, key is test_row_0/C:col10/1731230221039/Put/seqid=0 2024-11-10T09:17:01,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742058_1234 (size=13153) 2024-11-10T09:17:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:17:01,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:01,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T09:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:01,415 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:01,416 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:01,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:01,423 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/c1e01ef43ebb423e82380141350a8f43 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/c1e01ef43ebb423e82380141350a8f43 2024-11-10T09:17:01,429 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/C of 83bc899d24e43ccf5880b01ce9dff713 into c1e01ef43ebb423e82380141350a8f43(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:01,429 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:01,430 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/C, priority=13, startTime=1731230221185; duration=0sec 2024-11-10T09:17:01,430 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:01,430 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:C 2024-11-10T09:17:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:01,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:17:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:17:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230281602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230281602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230281603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230281603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,676 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/59f2a30f8e3e44b591a9c1284841ea56 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/59f2a30f8e3e44b591a9c1284841ea56 2024-11-10T09:17:01,686 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 83bc899d24e43ccf5880b01ce9dff713/B of 83bc899d24e43ccf5880b01ce9dff713 into 59f2a30f8e3e44b591a9c1284841ea56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:01,686 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:01,686 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., storeName=83bc899d24e43ccf5880b01ce9dff713/B, priority=13, startTime=1731230221185; duration=0sec 2024-11-10T09:17:01,687 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:01,687 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 83bc899d24e43ccf5880b01ce9dff713:B 2024-11-10T09:17:01,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:17:01,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:17:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:01,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:01,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c41e412cd0774b848693d51d4fc97473_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c41e412cd0774b848693d51d4fc97473_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:01,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/673c06423dde455cb0165dab3c000c02, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:01,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/673c06423dde455cb0165dab3c000c02 is 175, key is test_row_0/A:col10/1731230221207/Put/seqid=0 2024-11-10T09:17:01,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742059_1235 (size=31255) 2024-11-10T09:17:01,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/673c06423dde455cb0165dab3c000c02 2024-11-10T09:17:01,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5136130d7061481fa1121db6ba384669 is 50, key is test_row_0/B:col10/1731230221207/Put/seqid=0 2024-11-10T09:17:01,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742060_1236 (size=12301) 2024-11-10T09:17:01,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5136130d7061481fa1121db6ba384669 2024-11-10T09:17:01,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/a372cd790811411d8bd5e1f9f76d48de is 50, key is test_row_0/C:col10/1731230221207/Put/seqid=0 2024-11-10T09:17:01,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742061_1237 (size=12301) 2024-11-10T09:17:01,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:17:01,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:17:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:01,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57938 deadline: 1731230281907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57884 deadline: 1731230281907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57914 deadline: 1731230281917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57946 deadline: 1731230281918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,947 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-10T09:17:01,947 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:01,947 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,947 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57910 deadline: 1731230281947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:01,948 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-10T09:17:01,948 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:01,948 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,948 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:01,948 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 83bc899d24e43ccf5880b01ce9dff713 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:01,951 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-10T09:17:01,951 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-10T09:17:01,951 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:01,951 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:01,951 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,951 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,951 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,951 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:01,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:01,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:02,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:17:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:17:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:02,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:02,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:17:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:17:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:02,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:02,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/a372cd790811411d8bd5e1f9f76d48de 2024-11-10T09:17:02,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/673c06423dde455cb0165dab3c000c02 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/673c06423dde455cb0165dab3c000c02 2024-11-10T09:17:02,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/673c06423dde455cb0165dab3c000c02, entries=150, sequenceid=356, filesize=30.5 K 2024-11-10T09:17:02,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/5136130d7061481fa1121db6ba384669 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5136130d7061481fa1121db6ba384669 2024-11-10T09:17:02,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5136130d7061481fa1121db6ba384669, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T09:17:02,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/a372cd790811411d8bd5e1f9f76d48de as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/a372cd790811411d8bd5e1f9f76d48de 2024-11-10T09:17:02,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/a372cd790811411d8bd5e1f9f76d48de, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T09:17:02,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=100.63 KB/103050 for 83bc899d24e43ccf5880b01ce9dff713 in 1035ms, sequenceid=356, compaction requested=false 2024-11-10T09:17:02,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:02,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:02,340 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:17:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105fe5ef70c4e346dc9de788f572b5a569_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230221267/Put/seqid=0 2024-11-10T09:17:02,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742062_1238 (size=12454) 2024-11-10T09:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. as already flushing 2024-11-10T09:17:02,419 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-10T09:17:02,419 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-10T09:17:02,419 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:02,419 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:02,419 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,419 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,419 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,419 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:02,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:02,450 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-10T09:17:02,450 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-10T09:17:02,450 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:02,450 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:02,450 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,450 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,450 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,450 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:02,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:02,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:02,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:02,754 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105fe5ef70c4e346dc9de788f572b5a569_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105fe5ef70c4e346dc9de788f572b5a569_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/f505b7d869964c42a2cce81a8f5530dd, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:02,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/f505b7d869964c42a2cce81a8f5530dd is 175, key is test_row_0/A:col10/1731230221267/Put/seqid=0 2024-11-10T09:17:02,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742063_1239 (size=31255) 2024-11-10T09:17:03,161 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/f505b7d869964c42a2cce81a8f5530dd 2024-11-10T09:17:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/7720e21d44cd44d09eff9c6f8c11e8d2 is 50, key is test_row_0/B:col10/1731230221267/Put/seqid=0 2024-11-10T09:17:03,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742064_1240 (size=12301) 2024-11-10T09:17:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:03,574 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/7720e21d44cd44d09eff9c6f8c11e8d2 2024-11-10T09:17:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/2b74a578ce3d408cb3e6e38326f2756e is 50, key is test_row_0/C:col10/1731230221267/Put/seqid=0 2024-11-10T09:17:03,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742065_1241 (size=12301) 2024-11-10T09:17:03,592 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/2b74a578ce3d408cb3e6e38326f2756e 2024-11-10T09:17:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/f505b7d869964c42a2cce81a8f5530dd as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/f505b7d869964c42a2cce81a8f5530dd 2024-11-10T09:17:03,611 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/f505b7d869964c42a2cce81a8f5530dd, entries=150, sequenceid=377, filesize=30.5 K 2024-11-10T09:17:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/7720e21d44cd44d09eff9c6f8c11e8d2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/7720e21d44cd44d09eff9c6f8c11e8d2 2024-11-10T09:17:03,618 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/7720e21d44cd44d09eff9c6f8c11e8d2, entries=150, sequenceid=377, filesize=12.0 K 2024-11-10T09:17:03,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/2b74a578ce3d408cb3e6e38326f2756e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/2b74a578ce3d408cb3e6e38326f2756e 2024-11-10T09:17:03,625 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/2b74a578ce3d408cb3e6e38326f2756e, entries=150, sequenceid=377, filesize=12.0 K 2024-11-10T09:17:03,630 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 83bc899d24e43ccf5880b01ce9dff713 in 1290ms, sequenceid=377, compaction requested=true 2024-11-10T09:17:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 83bc899d24e43ccf5880b01ce9dff713: 2024-11-10T09:17:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T09:17:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T09:17:03,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T09:17:03,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2160 sec 2024-11-10T09:17:03,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2290 sec 2024-11-10T09:17:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:17:05,568 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:05,980 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-10T09:17:05,980 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:05,980 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:05,980 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:05,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2102 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2121 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 953 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2854 rows 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 946 2024-11-10T09:17:05,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2837 rows 2024-11-10T09:17:05,980 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:17:05,980 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:17:05,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:05,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:05,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:05,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:17:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:17:05,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230225984"}]},"ts":"1731230225984"} 2024-11-10T09:17:05,986 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:17:05,986 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:17:05,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:17:05,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, UNASSIGN}] 2024-11-10T09:17:05,989 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, UNASSIGN 2024-11-10T09:17:05,990 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:05,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, UNASSIGN because future has completed 2024-11-10T09:17:05,991 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:17:05,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:17:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:17:06,143 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 83bc899d24e43ccf5880b01ce9dff713, disabling compactions & flushes 2024-11-10T09:17:06,144 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. after waiting 0 ms 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:06,144 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 83bc899d24e43ccf5880b01ce9dff713 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=A 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=B 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:06,144 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 83bc899d24e43ccf5880b01ce9dff713, store=C 2024-11-10T09:17:06,145 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:06,152 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110973110a934ee455f80542d0167840b78_83bc899d24e43ccf5880b01ce9dff713 is 50, key is test_row_0/A:col10/1731230222428/Put/seqid=0 2024-11-10T09:17:06,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742066_1242 (size=12454) 2024-11-10T09:17:06,156 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:06,160 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110973110a934ee455f80542d0167840b78_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110973110a934ee455f80542d0167840b78_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:06,161 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b469de09d734dea8df5e6446ddca4b0, store: [table=TestAcidGuarantees family=A region=83bc899d24e43ccf5880b01ce9dff713] 2024-11-10T09:17:06,161 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b469de09d734dea8df5e6446ddca4b0 is 175, key is test_row_0/A:col10/1731230222428/Put/seqid=0 2024-11-10T09:17:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742067_1243 (size=31255) 2024-11-10T09:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:17:06,570 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b469de09d734dea8df5e6446ddca4b0 2024-11-10T09:17:06,584 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/f7066d87f7e14085948a0a5d8a7906f0 is 50, key is test_row_0/B:col10/1731230222428/Put/seqid=0 2024-11-10T09:17:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742068_1244 (size=12301) 2024-11-10T09:17:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:17:06,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:17:06,989 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/f7066d87f7e14085948a0a5d8a7906f0 2024-11-10T09:17:06,997 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/35668d1dc1f0455abae5f8b46094f1c3 is 50, key is test_row_0/C:col10/1731230222428/Put/seqid=0 2024-11-10T09:17:07,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742069_1245 (size=12301) 2024-11-10T09:17:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:17:07,402 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/35668d1dc1f0455abae5f8b46094f1c3 2024-11-10T09:17:07,408 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/A/9b469de09d734dea8df5e6446ddca4b0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b469de09d734dea8df5e6446ddca4b0 2024-11-10T09:17:07,414 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b469de09d734dea8df5e6446ddca4b0, entries=150, sequenceid=385, filesize=30.5 K 2024-11-10T09:17:07,415 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/B/f7066d87f7e14085948a0a5d8a7906f0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f7066d87f7e14085948a0a5d8a7906f0 2024-11-10T09:17:07,419 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f7066d87f7e14085948a0a5d8a7906f0, entries=150, sequenceid=385, filesize=12.0 K 2024-11-10T09:17:07,420 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/.tmp/C/35668d1dc1f0455abae5f8b46094f1c3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/35668d1dc1f0455abae5f8b46094f1c3 2024-11-10T09:17:07,424 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/35668d1dc1f0455abae5f8b46094f1c3, entries=150, sequenceid=385, filesize=12.0 K 2024-11-10T09:17:07,425 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 83bc899d24e43ccf5880b01ce9dff713 in 1281ms, sequenceid=385, compaction requested=true 2024-11-10T09:17:07,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4e399a0686b34ed4824085bd70eb638a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/92bf309a68f24de4ae0b6abc7fcc46ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6f432af1e4284918af2235496aec8fcb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/d82450d35099414291d2f7ba89676fe2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/243d3f6ccdd241119c1f9a2bbf854518, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4f7ed136d18c43f6bae080fd1273d729, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/65c31f411a544b1a896a91b08c0534c7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e] to archive 2024-11-10T09:17:07,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:17:07,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/c173208aa56a433496837422884fae08 2024-11-10T09:17:07,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/b94430202ff44170bda51b885ee55168 2024-11-10T09:17:07,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b270949f9df4d6db6c1106c8ff4e676 2024-11-10T09:17:07,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4e399a0686b34ed4824085bd70eb638a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4e399a0686b34ed4824085bd70eb638a 2024-11-10T09:17:07,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/50b82b526ed0403eaa46f4b565543bc6 2024-11-10T09:17:07,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/92bf309a68f24de4ae0b6abc7fcc46ba to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/92bf309a68f24de4ae0b6abc7fcc46ba 2024-11-10T09:17:07,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/733fc9bbb6264bb6abeea500da2b144d 2024-11-10T09:17:07,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/3781ef5da5eb4f158b5a796d6bdf1850 2024-11-10T09:17:07,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/2eb8737e9214420bb3118f43396f58c6 2024-11-10T09:17:07,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6f432af1e4284918af2235496aec8fcb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6f432af1e4284918af2235496aec8fcb 2024-11-10T09:17:07,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/00f469d3a6684f4c9b6447531f38b2c3 2024-11-10T09:17:07,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/d82450d35099414291d2f7ba89676fe2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/d82450d35099414291d2f7ba89676fe2 2024-11-10T09:17:07,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1c8ccacc98334c7ea2f815f1849f9159 2024-11-10T09:17:07,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/56a7dc90dc6f4190ba8351af5e2c9b0b 2024-11-10T09:17:07,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/a418b08908d84e289ea783d48a9ec46f 2024-11-10T09:17:07,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/243d3f6ccdd241119c1f9a2bbf854518 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/243d3f6ccdd241119c1f9a2bbf854518 2024-11-10T09:17:07,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/0e47f92a69284fe0b480faa23de66d33 2024-11-10T09:17:07,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/8f3e3cc23ba4444987c8cf2e55aaa959 2024-11-10T09:17:07,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/352ea9e8b26547b3854992838f50f451 2024-11-10T09:17:07,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4f7ed136d18c43f6bae080fd1273d729 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/4f7ed136d18c43f6bae080fd1273d729 2024-11-10T09:17:07,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/1720c2537f80407f8106084629e5a950 2024-11-10T09:17:07,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/6fe320c1b49f404abe00c2c0a324bbf7 2024-11-10T09:17:07,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/65c31f411a544b1a896a91b08c0534c7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/65c31f411a544b1a896a91b08c0534c7 2024-11-10T09:17:07,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/84447571451f4dac8002177be8e63d89 2024-11-10T09:17:07,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/222a1488792d41e2b3a6270099a61d8e 2024-11-10T09:17:07,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/1121b1c952a941c183111ea65bc3e1fc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4636dbfc66c84b2bb009b66d196d1494, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f87896ab9d814d2283a8d2db5e7514d4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5acae3625d274e88849e30a437ec052b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/c3e256aa75a5477bb88e1b3cba03b0bb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/fa9e30ea8dd44e22ab866e22bdf382ab, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ae7a165a69df4d64ab8a16de396abc97, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/e5cfcb06aa994703b7f52e30e6c2ffdd, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6b0fd52a5e8d4e308aaa0835e56f5944, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/8e6e463db76344a68f962981dbafbcbf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/0444aab0f76047c988585389b3140995, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/69b53cd37d7f4f8bb5aad3b09c216c5a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a31dc33f4f3140b0a211cff85bd7a632, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/70c9ac386a034be09d68748bb5cafcd8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/38ce3af92f7743f38ca8ff9ebcf230bc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/b907ea51d1b740f8a6e3a54327e29ed8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a3dad0dbc8954fc39dced6e8903bf404, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ef1803cac415447485ae698edc768f39, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/bb0cfbf6e81b4f368888897e905a0b1b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/781ac33603924193a2c855208e1db128, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5790d3da37d647e48a6d5688e4f19dbf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4b88de2242e54b1ba534317655f17e0d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6408a83e8fca408aa4340133708e3e44, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/3c4800d5a6ef4adda8b33161b1e030c5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/954347ea1c324aab90a25ad412816fd3] to archive 2024-11-10T09:17:07,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:17:07,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/1121b1c952a941c183111ea65bc3e1fc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/1121b1c952a941c183111ea65bc3e1fc 2024-11-10T09:17:07,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4636dbfc66c84b2bb009b66d196d1494 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4636dbfc66c84b2bb009b66d196d1494 2024-11-10T09:17:07,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f87896ab9d814d2283a8d2db5e7514d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f87896ab9d814d2283a8d2db5e7514d4 2024-11-10T09:17:07,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5acae3625d274e88849e30a437ec052b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5acae3625d274e88849e30a437ec052b 2024-11-10T09:17:07,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/c3e256aa75a5477bb88e1b3cba03b0bb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/c3e256aa75a5477bb88e1b3cba03b0bb 2024-11-10T09:17:07,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/fa9e30ea8dd44e22ab866e22bdf382ab to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/fa9e30ea8dd44e22ab866e22bdf382ab 2024-11-10T09:17:07,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ae7a165a69df4d64ab8a16de396abc97 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ae7a165a69df4d64ab8a16de396abc97 2024-11-10T09:17:07,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/e5cfcb06aa994703b7f52e30e6c2ffdd to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/e5cfcb06aa994703b7f52e30e6c2ffdd 2024-11-10T09:17:07,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6b0fd52a5e8d4e308aaa0835e56f5944 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6b0fd52a5e8d4e308aaa0835e56f5944 2024-11-10T09:17:07,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/8e6e463db76344a68f962981dbafbcbf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/8e6e463db76344a68f962981dbafbcbf 2024-11-10T09:17:07,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/0444aab0f76047c988585389b3140995 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/0444aab0f76047c988585389b3140995 2024-11-10T09:17:07,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/69b53cd37d7f4f8bb5aad3b09c216c5a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/69b53cd37d7f4f8bb5aad3b09c216c5a 2024-11-10T09:17:07,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a31dc33f4f3140b0a211cff85bd7a632 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a31dc33f4f3140b0a211cff85bd7a632 2024-11-10T09:17:07,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/70c9ac386a034be09d68748bb5cafcd8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/70c9ac386a034be09d68748bb5cafcd8 2024-11-10T09:17:07,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/38ce3af92f7743f38ca8ff9ebcf230bc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/38ce3af92f7743f38ca8ff9ebcf230bc 2024-11-10T09:17:07,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/b907ea51d1b740f8a6e3a54327e29ed8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/b907ea51d1b740f8a6e3a54327e29ed8 2024-11-10T09:17:07,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a3dad0dbc8954fc39dced6e8903bf404 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/a3dad0dbc8954fc39dced6e8903bf404 2024-11-10T09:17:07,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ef1803cac415447485ae698edc768f39 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/ef1803cac415447485ae698edc768f39 2024-11-10T09:17:07,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/bb0cfbf6e81b4f368888897e905a0b1b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/bb0cfbf6e81b4f368888897e905a0b1b 2024-11-10T09:17:07,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/781ac33603924193a2c855208e1db128 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/781ac33603924193a2c855208e1db128 2024-11-10T09:17:07,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5790d3da37d647e48a6d5688e4f19dbf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5790d3da37d647e48a6d5688e4f19dbf 2024-11-10T09:17:07,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4b88de2242e54b1ba534317655f17e0d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/4b88de2242e54b1ba534317655f17e0d 2024-11-10T09:17:07,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6408a83e8fca408aa4340133708e3e44 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/6408a83e8fca408aa4340133708e3e44 2024-11-10T09:17:07,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/3c4800d5a6ef4adda8b33161b1e030c5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/3c4800d5a6ef4adda8b33161b1e030c5 2024-11-10T09:17:07,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/954347ea1c324aab90a25ad412816fd3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/954347ea1c324aab90a25ad412816fd3 2024-11-10T09:17:07,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/d13319252f044a6fbb190c02e88eb43b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/e374cb2a201c4a8c946baa1199f028b5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/63751e38933a4914b4f1e910163100e3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/25e3531dedca419fa00ddd4a6c2c7ac8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/51b7863cc4f045b48a024a308ae3a7ec, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b42c3761280742f3b633d691cb11a846, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/36ab9ce24b5d498aa5000cff3265938a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/831c1aac2169420fa3ff40ee2cf443ae, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/0b7da559dcbf4f0fa818e60b9949488f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/85a8a4340e784451bc051525e8a04ca7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f109959df005451180b3583bf9cef135, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f73c588ffd6f4210b22456d2ffb74dbb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/7fdac0a293f3438bb95c83236e44111d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/732d8ba7d35f4b8b942e84c1add4e618, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/56048900ec084d3a8305f932ce5d0fa5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/287c7983114b49c7beaa6d3ed5ff3f75, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/3c6307650253407388b5652a6a4a5335, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1dfd70c65d8e49da97923ac62ce41933, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f8750197c20a4df1817f78edaabac43d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1c23f7000df640b599bb0b47a6b9eb84, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/facb4e2bb0bd43a2befb102e9aa9567b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/9de752660a5c47cf930a1c191829d89c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/090f1c632ab6404a9f3482cc42e6864f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/8455797446fc40b19a51211051200a35, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b48918c597e94dce83b2315a1ba0f58a] to archive 2024-11-10T09:17:07,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:17:07,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/d13319252f044a6fbb190c02e88eb43b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/d13319252f044a6fbb190c02e88eb43b 2024-11-10T09:17:07,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/e374cb2a201c4a8c946baa1199f028b5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/e374cb2a201c4a8c946baa1199f028b5 2024-11-10T09:17:07,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/63751e38933a4914b4f1e910163100e3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/63751e38933a4914b4f1e910163100e3 2024-11-10T09:17:07,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/25e3531dedca419fa00ddd4a6c2c7ac8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/25e3531dedca419fa00ddd4a6c2c7ac8 2024-11-10T09:17:07,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/51b7863cc4f045b48a024a308ae3a7ec to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/51b7863cc4f045b48a024a308ae3a7ec 2024-11-10T09:17:07,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b42c3761280742f3b633d691cb11a846 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b42c3761280742f3b633d691cb11a846 2024-11-10T09:17:07,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/36ab9ce24b5d498aa5000cff3265938a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/36ab9ce24b5d498aa5000cff3265938a 2024-11-10T09:17:07,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/831c1aac2169420fa3ff40ee2cf443ae to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/831c1aac2169420fa3ff40ee2cf443ae 2024-11-10T09:17:07,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/0b7da559dcbf4f0fa818e60b9949488f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/0b7da559dcbf4f0fa818e60b9949488f 2024-11-10T09:17:07,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/85a8a4340e784451bc051525e8a04ca7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/85a8a4340e784451bc051525e8a04ca7 2024-11-10T09:17:07,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f109959df005451180b3583bf9cef135 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f109959df005451180b3583bf9cef135 2024-11-10T09:17:07,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f73c588ffd6f4210b22456d2ffb74dbb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f73c588ffd6f4210b22456d2ffb74dbb 2024-11-10T09:17:07,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/7fdac0a293f3438bb95c83236e44111d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/7fdac0a293f3438bb95c83236e44111d 2024-11-10T09:17:07,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/732d8ba7d35f4b8b942e84c1add4e618 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/732d8ba7d35f4b8b942e84c1add4e618 2024-11-10T09:17:07,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/56048900ec084d3a8305f932ce5d0fa5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/56048900ec084d3a8305f932ce5d0fa5 2024-11-10T09:17:07,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/287c7983114b49c7beaa6d3ed5ff3f75 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/287c7983114b49c7beaa6d3ed5ff3f75 2024-11-10T09:17:07,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/3c6307650253407388b5652a6a4a5335 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/3c6307650253407388b5652a6a4a5335 2024-11-10T09:17:07,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1dfd70c65d8e49da97923ac62ce41933 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1dfd70c65d8e49da97923ac62ce41933 2024-11-10T09:17:07,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f8750197c20a4df1817f78edaabac43d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/f8750197c20a4df1817f78edaabac43d 2024-11-10T09:17:07,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1c23f7000df640b599bb0b47a6b9eb84 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/1c23f7000df640b599bb0b47a6b9eb84 2024-11-10T09:17:07,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/facb4e2bb0bd43a2befb102e9aa9567b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/facb4e2bb0bd43a2befb102e9aa9567b 2024-11-10T09:17:07,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/9de752660a5c47cf930a1c191829d89c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/9de752660a5c47cf930a1c191829d89c 2024-11-10T09:17:07,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/090f1c632ab6404a9f3482cc42e6864f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/090f1c632ab6404a9f3482cc42e6864f 2024-11-10T09:17:07,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/8455797446fc40b19a51211051200a35 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/8455797446fc40b19a51211051200a35 2024-11-10T09:17:07,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b48918c597e94dce83b2315a1ba0f58a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/b48918c597e94dce83b2315a1ba0f58a 2024-11-10T09:17:07,512 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/recovered.edits/388.seqid, newMaxSeqId=388, maxSeqId=4 2024-11-10T09:17:07,512 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. 2024-11-10T09:17:07,513 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 83bc899d24e43ccf5880b01ce9dff713: Waiting for close lock at 1731230226144Running coprocessor pre-close hooks at 1731230226144Disabling compacts and flushes for region at 1731230226144Disabling writes for close at 1731230226144Obtaining lock to block concurrent updates at 1731230226144Preparing flush snapshotting stores in 83bc899d24e43ccf5880b01ce9dff713 at 1731230226144Finished memstore snapshotting TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731230226145 (+1 ms)Flushing stores of TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713. at 1731230226145Flushing 83bc899d24e43ccf5880b01ce9dff713/A: creating writer at 1731230226145Flushing mob file 83bc899d24e43ccf5880b01ce9dff713/A: appending metadata at 1731230226152 (+7 ms)Flushing mob file 83bc899d24e43ccf5880b01ce9dff713/A: closing flushed file at 1731230226152Flushing 83bc899d24e43ccf5880b01ce9dff713/A: appending metadata at 1731230226161 (+9 ms)Flushing 83bc899d24e43ccf5880b01ce9dff713/A: closing flushed file at 1731230226161Flushing 83bc899d24e43ccf5880b01ce9dff713/B: creating writer at 1731230226578 (+417 ms)Flushing 83bc899d24e43ccf5880b01ce9dff713/B: appending metadata at 1731230226583 (+5 ms)Flushing 83bc899d24e43ccf5880b01ce9dff713/B: closing flushed file at 1731230226583Flushing 83bc899d24e43ccf5880b01ce9dff713/C: creating writer at 1731230226993 (+410 ms)Flushing 83bc899d24e43ccf5880b01ce9dff713/C: appending metadata at 1731230226996 (+3 ms)Flushing 83bc899d24e43ccf5880b01ce9dff713/C: closing flushed file at 1731230226996Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@693d57cd: reopening flushed file at 1731230227407 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79ddc660: reopening flushed file at 1731230227414 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11b8e841: reopening flushed file at 1731230227419 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 83bc899d24e43ccf5880b01ce9dff713 in 1281ms, sequenceid=385, compaction requested=true at 1731230227425 (+6 ms)Writing region close event to WAL at 1731230227509 (+84 ms)Running coprocessor post-close hooks at 1731230227512 (+3 ms)Closed at 1731230227512 2024-11-10T09:17:07,514 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:07,514 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=83bc899d24e43ccf5880b01ce9dff713, regionState=CLOSED 2024-11-10T09:17:07,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:17:07,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T09:17:07,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 83bc899d24e43ccf5880b01ce9dff713, server=802346a334f5,38513,1731230168538 in 1.5250 sec 2024-11-10T09:17:07,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-10T09:17:07,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=83bc899d24e43ccf5880b01ce9dff713, UNASSIGN in 1.5300 sec 2024-11-10T09:17:07,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T09:17:07,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-10T09:17:07,522 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230227522"}]},"ts":"1731230227522"} 2024-11-10T09:17:07,523 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:17:07,523 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:17:07,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5420 sec 2024-11-10T09:17:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:17:08,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:08,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:17:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,131 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T09:17:08,132 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,136 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/recovered.edits] 2024-11-10T09:17:08,147 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/48d48a68666e4ab99a4c9daa47b5f801 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/48d48a68666e4ab99a4c9daa47b5f801 2024-11-10T09:17:08,150 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/673c06423dde455cb0165dab3c000c02 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/673c06423dde455cb0165dab3c000c02 2024-11-10T09:17:08,152 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b469de09d734dea8df5e6446ddca4b0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/9b469de09d734dea8df5e6446ddca4b0 2024-11-10T09:17:08,154 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/f505b7d869964c42a2cce81a8f5530dd to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/A/f505b7d869964c42a2cce81a8f5530dd 2024-11-10T09:17:08,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:17:08,157 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5136130d7061481fa1121db6ba384669 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/5136130d7061481fa1121db6ba384669 2024-11-10T09:17:08,158 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/59f2a30f8e3e44b591a9c1284841ea56 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/59f2a30f8e3e44b591a9c1284841ea56 2024-11-10T09:17:08,160 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/7720e21d44cd44d09eff9c6f8c11e8d2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/7720e21d44cd44d09eff9c6f8c11e8d2 2024-11-10T09:17:08,161 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f7066d87f7e14085948a0a5d8a7906f0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/B/f7066d87f7e14085948a0a5d8a7906f0 2024-11-10T09:17:08,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/2b74a578ce3d408cb3e6e38326f2756e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/2b74a578ce3d408cb3e6e38326f2756e 2024-11-10T09:17:08,165 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/35668d1dc1f0455abae5f8b46094f1c3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/35668d1dc1f0455abae5f8b46094f1c3 2024-11-10T09:17:08,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/a372cd790811411d8bd5e1f9f76d48de to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/a372cd790811411d8bd5e1f9f76d48de 2024-11-10T09:17:08,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/c1e01ef43ebb423e82380141350a8f43 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/C/c1e01ef43ebb423e82380141350a8f43 2024-11-10T09:17:08,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/recovered.edits/388.seqid to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713/recovered.edits/388.seqid 2024-11-10T09:17:08,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,169 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:17:08,169 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:17:08,170 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T09:17:08,173 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111010b7f4be771045158e127c0fd0f39391_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111010b7f4be771045158e127c0fd0f39391_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,174 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101bfd754956454b3e911818bc8bc15e1e_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101bfd754956454b3e911818bc8bc15e1e_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,175 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102564ab2feca540c2b545694173a1b376_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102564ab2feca540c2b545694173a1b376_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,176 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103226619003754993be239750f04d76d3_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103226619003754993be239750f04d76d3_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,178 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032bb03ec49ea4479b844ecae07a24521_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032bb03ec49ea4479b844ecae07a24521_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,179 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111038b99edae9e24026b17c10cae93fc0c7_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111038b99edae9e24026b17c10cae93fc0c7_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,180 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103f1f62b641d9430fb62ed138eb8f3bf1_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103f1f62b641d9430fb62ed138eb8f3bf1_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,182 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104747b5795f5741e58f12a7c4f57b398e_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104747b5795f5741e58f12a7c4f57b398e_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,183 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105475eeef21b848f1a32151124e265639_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105475eeef21b848f1a32151124e265639_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,184 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105fe5ef70c4e346dc9de788f572b5a569_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105fe5ef70c4e346dc9de788f572b5a569_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,186 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061f54f5e513b4ff0b5866f2b7fda1700_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061f54f5e513b4ff0b5866f2b7fda1700_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,187 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108499c40f8b094483b9ca80bf9eeb9ab7_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108499c40f8b094483b9ca80bf9eeb9ab7_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,188 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110973110a934ee455f80542d0167840b78_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110973110a934ee455f80542d0167840b78_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,189 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c18d70b44f344a7eae4b64e633165463_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c18d70b44f344a7eae4b64e633165463_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,190 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c41e412cd0774b848693d51d4fc97473_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c41e412cd0774b848693d51d4fc97473_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,191 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cfb8217c879645fdb3b7a3b44a9ea729_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cfb8217c879645fdb3b7a3b44a9ea729_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,192 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110da1e26c761f849bfa455d264e58cec7f_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110da1e26c761f849bfa455d264e58cec7f_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,193 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd8298a750d84b5ca0101150e83a2924_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd8298a750d84b5ca0101150e83a2924_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,194 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e915a7a7b8c94f9fb6e3675ab3686051_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e915a7a7b8c94f9fb6e3675ab3686051_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,195 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ef9326d685d34dde994b8379c346943b_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ef9326d685d34dde994b8379c346943b_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,196 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fcb2e1367fc74e289e947248b48f7ce4_83bc899d24e43ccf5880b01ce9dff713 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fcb2e1367fc74e289e947248b48f7ce4_83bc899d24e43ccf5880b01ce9dff713 2024-11-10T09:17:08,196 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:17:08,198 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,200 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:17:08,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:17:08,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:17:08,204 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230228204"}]},"ts":"9223372036854775807"} 2024-11-10T09:17:08,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:17:08,206 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 83bc899d24e43ccf5880b01ce9dff713, NAME => 'TestAcidGuarantees,,1731230200678.83bc899d24e43ccf5880b01ce9dff713.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:17:08,207 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:17:08,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230228207"}]},"ts":"9223372036854775807"} 2024-11-10T09:17:08,209 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:17:08,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-10T09:17:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T09:17:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:17:08,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:08,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 249) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-704730897_22 at /127.0.0.1:54434 [Waiting for operation #578] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_381167618_22 at /127.0.0.1:54440 [Waiting for operation #576] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_381167618_22 at /127.0.0.1:55438 [Waiting for operation #888] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-704730897_22 at /127.0.0.1:54408 [Waiting for operation #604] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=826 (was 760) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1857 (was 2747) 2024-11-10T09:17:08,262 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=826, ProcessCount=11, AvailableMemoryMB=1857 2024-11-10T09:17:08,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:17:08,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:17:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:08,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:17:08,266 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:08,266 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-10T09:17:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T09:17:08,267 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:17:08,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742070_1246 (size=1061) 2024-11-10T09:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T09:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T09:17:08,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:17:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742071_1247 (size=53) 2024-11-10T09:17:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T09:17:09,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:17:09,082 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b031bdc33b2db2f1f99a0ca67fb65c56, disabling compactions & flushes 2024-11-10T09:17:09,082 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,082 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,082 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. after waiting 0 ms 2024-11-10T09:17:09,082 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,082 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,082 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b031bdc33b2db2f1f99a0ca67fb65c56: Waiting for close lock at 1731230229082Disabling compacts and flushes for region at 1731230229082Disabling writes for close at 1731230229082Writing region close event to WAL at 1731230229082Closed at 1731230229082 2024-11-10T09:17:09,083 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:17:09,083 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230229083"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230229083"}]},"ts":"1731230229083"} 2024-11-10T09:17:09,085 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:17:09,085 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:17:09,086 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230229085"}]},"ts":"1731230229085"} 2024-11-10T09:17:09,087 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:17:09,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, ASSIGN}] 2024-11-10T09:17:09,088 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, ASSIGN 2024-11-10T09:17:09,089 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, ASSIGN; state=OFFLINE, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=false 2024-11-10T09:17:09,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b031bdc33b2db2f1f99a0ca67fb65c56, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:09,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, ASSIGN because future has completed 2024-11-10T09:17:09,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:17:09,401 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,402 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:17:09,402 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,402 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:17:09,403 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,403 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,404 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,406 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:09,406 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b031bdc33b2db2f1f99a0ca67fb65c56 columnFamilyName A 2024-11-10T09:17:09,407 DEBUG [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T09:17:09,407 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.HStore(327): Store=b031bdc33b2db2f1f99a0ca67fb65c56/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:09,408 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,409 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:09,409 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b031bdc33b2db2f1f99a0ca67fb65c56 columnFamilyName B 2024-11-10T09:17:09,409 DEBUG [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:09,410 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.HStore(327): Store=b031bdc33b2db2f1f99a0ca67fb65c56/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:09,410 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,411 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:09,411 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b031bdc33b2db2f1f99a0ca67fb65c56 columnFamilyName C 2024-11-10T09:17:09,411 DEBUG [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:09,411 INFO [StoreOpener-b031bdc33b2db2f1f99a0ca67fb65c56-1 {}] regionserver.HStore(327): Store=b031bdc33b2db2f1f99a0ca67fb65c56/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:09,412 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,412 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,412 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,413 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,413 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,413 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,414 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:17:09,415 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,416 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:17:09,417 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened b031bdc33b2db2f1f99a0ca67fb65c56; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65516537, jitterRate=-0.023727521300315857}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:17:09,417 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:09,418 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for b031bdc33b2db2f1f99a0ca67fb65c56: Running coprocessor pre-open hook at 1731230229403Writing region info on filesystem at 1731230229403Initializing all the Stores at 1731230229404 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230229404Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230229404Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230229404Cleaning up temporary data from old regions at 1731230229413 (+9 ms)Running coprocessor post-open hooks at 1731230229417 (+4 ms)Region opened successfully at 1731230229418 (+1 ms) 2024-11-10T09:17:09,419 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., pid=63, masterSystemTime=1731230229397 2024-11-10T09:17:09,420 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,420 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:09,421 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b031bdc33b2db2f1f99a0ca67fb65c56, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:09,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:17:09,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-10T09:17:09,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 in 179 msec 2024-11-10T09:17:09,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-10T09:17:09,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, ASSIGN in 338 msec 2024-11-10T09:17:09,427 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:17:09,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230229427"}]},"ts":"1731230229427"} 2024-11-10T09:17:09,428 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:17:09,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:17:09,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-10T09:17:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T09:17:10,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:10,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3d806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfd0d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,421 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f94bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,421 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0f0acc 2024-11-10T09:17:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:17:10,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:17:10,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e5bf58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184b0249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1820ccb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b24a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,430 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dacfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,431 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4c46a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e66c663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,434 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f109cc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec1ffcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7791dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215556cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9c64b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51107695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,441 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d26ce25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ca213b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a987c8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,443 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d650c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,445 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60cfd495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a60f526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5c9dcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,448 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd078e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,449 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,451 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515ac970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,453 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5374e7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b33a7f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a38ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:10,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:10,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3b8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:10,457 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c2a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:10,458 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,458 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,458 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,458 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,458 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,458 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,459 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,459 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,460 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,461 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:10,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,461 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:10,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:10,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-10T09:17:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,465 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,468 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:10,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:10,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:17:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:10,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:17:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:10,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/436928103a154e80a3274aa51a7f606b is 50, key is test_row_0/A:col10/1731230230475/Put/seqid=0 2024-11-10T09:17:10,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230290540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230290542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230290543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230290555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230290555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:10,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742072_1248 (size=12001) 2024-11-10T09:17:10,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:10,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230290657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230290657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230290657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230290667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230290678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:10,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:10,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230290867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230290867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230290867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230290877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230290888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:10,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:10,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:10,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/436928103a154e80a3274aa51a7f606b 2024-11-10T09:17:11,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/1c62ea4e3e9a487489fd52e5f0207d50 is 50, key is test_row_0/B:col10/1731230230475/Put/seqid=0 2024-11-10T09:17:11,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742073_1249 (size=12001) 2024-11-10T09:17:11,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/1c62ea4e3e9a487489fd52e5f0207d50 2024-11-10T09:17:11,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/fc6fba309f6540dcb8748b51f053b55f is 50, key is test_row_0/C:col10/1731230230475/Put/seqid=0 2024-11-10T09:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:11,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742074_1250 (size=12001) 2024-11-10T09:17:11,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/fc6fba309f6540dcb8748b51f053b55f 2024-11-10T09:17:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/436928103a154e80a3274aa51a7f606b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/436928103a154e80a3274aa51a7f606b 2024-11-10T09:17:11,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/436928103a154e80a3274aa51a7f606b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:17:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/1c62ea4e3e9a487489fd52e5f0207d50 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/1c62ea4e3e9a487489fd52e5f0207d50 2024-11-10T09:17:11,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/1c62ea4e3e9a487489fd52e5f0207d50, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:17:11,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/fc6fba309f6540dcb8748b51f053b55f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/fc6fba309f6540dcb8748b51f053b55f 2024-11-10T09:17:11,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/fc6fba309f6540dcb8748b51f053b55f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:17:11,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for b031bdc33b2db2f1f99a0ca67fb65c56 in 672ms, sequenceid=15, compaction requested=false 2024-11-10T09:17:11,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:11,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:17:11,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:11,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:11,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:11,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:11,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:11,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:11,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8d169065e7de46cca79db6f1af3b8c4a is 50, key is test_row_0/A:col10/1731230230537/Put/seqid=0 2024-11-10T09:17:11,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230291202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230291209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230291209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230291211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230291211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742075_1251 (size=14341) 2024-11-10T09:17:11,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8d169065e7de46cca79db6f1af3b8c4a 2024-11-10T09:17:11,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/427dea98fecc4672bdaded703a76f6ad is 50, key is test_row_0/B:col10/1731230230537/Put/seqid=0 2024-11-10T09:17:11,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:11,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:11,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742076_1252 (size=12001) 2024-11-10T09:17:11,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/427dea98fecc4672bdaded703a76f6ad 2024-11-10T09:17:11,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8a16aab773f04d088c19d2bff8cb9275 is 50, key is test_row_0/C:col10/1731230230537/Put/seqid=0 2024-11-10T09:17:11,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742077_1253 (size=12001) 2024-11-10T09:17:11,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8a16aab773f04d088c19d2bff8cb9275 2024-11-10T09:17:11,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8d169065e7de46cca79db6f1af3b8c4a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8d169065e7de46cca79db6f1af3b8c4a 2024-11-10T09:17:11,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8d169065e7de46cca79db6f1af3b8c4a, entries=200, sequenceid=35, filesize=14.0 K 2024-11-10T09:17:11,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/427dea98fecc4672bdaded703a76f6ad as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/427dea98fecc4672bdaded703a76f6ad 2024-11-10T09:17:11,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/427dea98fecc4672bdaded703a76f6ad, entries=150, sequenceid=35, filesize=11.7 K 2024-11-10T09:17:11,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8a16aab773f04d088c19d2bff8cb9275 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8a16aab773f04d088c19d2bff8cb9275 2024-11-10T09:17:11,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8a16aab773f04d088c19d2bff8cb9275, entries=150, sequenceid=35, filesize=11.7 K 2024-11-10T09:17:11,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b031bdc33b2db2f1f99a0ca67fb65c56 in 123ms, sequenceid=35, compaction requested=false 2024-11-10T09:17:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:11,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:17:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:11,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/9acec32ccd5442bfabe9bf5c2415bc9c is 50, key is test_row_0/A:col10/1731230231205/Put/seqid=0 2024-11-10T09:17:11,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742078_1254 (size=14341) 2024-11-10T09:17:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230291344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230291348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230291351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230291352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230291354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230291457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230291458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230291467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230291467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230291467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230291667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230291667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230291677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230291677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230291677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/9acec32ccd5442bfabe9bf5c2415bc9c 2024-11-10T09:17:11,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/95e9c377aa954bc586f1c5b0b1cb415a is 50, key is test_row_0/B:col10/1731230231205/Put/seqid=0 2024-11-10T09:17:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742079_1255 (size=12001) 2024-11-10T09:17:11,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:11,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:11,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:11,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230291977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230291977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230291987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230291987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230291987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:12,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/95e9c377aa954bc586f1c5b0b1cb415a 2024-11-10T09:17:12,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8e7a01e55a6a45d69699a351e5b0237c is 50, key is test_row_0/C:col10/1731230231205/Put/seqid=0 2024-11-10T09:17:12,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742080_1256 (size=12001) 2024-11-10T09:17:12,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230292487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:12,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230292487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:12,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230292498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:12,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230292498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:12,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230292498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:12,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:12,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8e7a01e55a6a45d69699a351e5b0237c 2024-11-10T09:17:12,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/9acec32ccd5442bfabe9bf5c2415bc9c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/9acec32ccd5442bfabe9bf5c2415bc9c 2024-11-10T09:17:12,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/9acec32ccd5442bfabe9bf5c2415bc9c, entries=200, sequenceid=49, filesize=14.0 K 2024-11-10T09:17:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/95e9c377aa954bc586f1c5b0b1cb415a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/95e9c377aa954bc586f1c5b0b1cb415a 2024-11-10T09:17:12,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/95e9c377aa954bc586f1c5b0b1cb415a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:17:12,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8e7a01e55a6a45d69699a351e5b0237c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e7a01e55a6a45d69699a351e5b0237c 2024-11-10T09:17:12,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e7a01e55a6a45d69699a351e5b0237c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:17:12,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1349ms, sequenceid=49, compaction requested=true 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:12,670 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:12,670 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:12,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:12,672 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:12,672 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:12,672 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,672 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/436928103a154e80a3274aa51a7f606b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8d169065e7de46cca79db6f1af3b8c4a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/9acec32ccd5442bfabe9bf5c2415bc9c] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=39.7 K 2024-11-10T09:17:12,672 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:12,672 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:12,673 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,673 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/1c62ea4e3e9a487489fd52e5f0207d50, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/427dea98fecc4672bdaded703a76f6ad, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/95e9c377aa954bc586f1c5b0b1cb415a] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.2 K 2024-11-10T09:17:12,673 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 436928103a154e80a3274aa51a7f606b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230230475 2024-11-10T09:17:12,673 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c62ea4e3e9a487489fd52e5f0207d50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230230475 2024-11-10T09:17:12,674 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 427dea98fecc4672bdaded703a76f6ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731230230537 2024-11-10T09:17:12,674 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d169065e7de46cca79db6f1af3b8c4a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731230230537 2024-11-10T09:17:12,674 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e9c377aa954bc586f1c5b0b1cb415a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230231194 2024-11-10T09:17:12,674 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9acec32ccd5442bfabe9bf5c2415bc9c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230231194 2024-11-10T09:17:12,691 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:12,692 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/98de9732995543c48182531010212b21 is 50, key is test_row_0/A:col10/1731230231205/Put/seqid=0 2024-11-10T09:17:12,694 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:12,694 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/d87c38488db24936978ff5647a8259c0 is 50, key is test_row_0/B:col10/1731230231205/Put/seqid=0 2024-11-10T09:17:12,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742082_1258 (size=12104) 2024-11-10T09:17:12,749 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/d87c38488db24936978ff5647a8259c0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/d87c38488db24936978ff5647a8259c0 2024-11-10T09:17:12,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742081_1257 (size=12104) 2024-11-10T09:17:12,756 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into d87c38488db24936978ff5647a8259c0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:12,756 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:12,756 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230232670; duration=0sec 2024-11-10T09:17:12,756 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:12,756 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:12,756 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:12,758 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:12,758 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:12,759 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,760 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/fc6fba309f6540dcb8748b51f053b55f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8a16aab773f04d088c19d2bff8cb9275, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e7a01e55a6a45d69699a351e5b0237c] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.2 K 2024-11-10T09:17:12,760 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/98de9732995543c48182531010212b21 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/98de9732995543c48182531010212b21 2024-11-10T09:17:12,760 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting fc6fba309f6540dcb8748b51f053b55f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230230475 2024-11-10T09:17:12,761 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a16aab773f04d088c19d2bff8cb9275, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731230230537 2024-11-10T09:17:12,762 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7a01e55a6a45d69699a351e5b0237c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230231194 2024-11-10T09:17:12,766 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into 98de9732995543c48182531010212b21(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:12,766 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:12,766 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230232670; duration=0sec 2024-11-10T09:17:12,766 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:12,767 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:12,781 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:12,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T09:17:12,782 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/96194392d1f1430c86bdf432d5a32691 is 50, key is test_row_0/C:col10/1731230231205/Put/seqid=0 2024-11-10T09:17:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:17:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:12,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/d0e9d612008e41d695d57302cc312c97 is 50, key is test_row_0/A:col10/1731230231351/Put/seqid=0 2024-11-10T09:17:12,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742083_1259 (size=12104) 2024-11-10T09:17:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742084_1260 (size=12001) 2024-11-10T09:17:13,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:17:13,230 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/96194392d1f1430c86bdf432d5a32691 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/96194392d1f1430c86bdf432d5a32691 2024-11-10T09:17:13,235 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/d0e9d612008e41d695d57302cc312c97 2024-11-10T09:17:13,243 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into 96194392d1f1430c86bdf432d5a32691(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:13,243 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:13,243 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230232670; duration=0sec 2024-11-10T09:17:13,243 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:13,243 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/0ee55efd98214997b84ef388ef676cc1 is 50, key is test_row_0/B:col10/1731230231351/Put/seqid=0 2024-11-10T09:17:13,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742085_1261 (size=12001) 2024-11-10T09:17:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:13,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230293517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230293517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,521 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230293517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,522 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,522 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230293517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230293521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230293627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230293630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,662 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/0ee55efd98214997b84ef388ef676cc1 2024-11-10T09:17:13,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/300d024255af45d686ccdef3c4866c81 is 50, key is test_row_0/C:col10/1731230231351/Put/seqid=0 2024-11-10T09:17:13,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742086_1262 (size=12001) 2024-11-10T09:17:13,705 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/300d024255af45d686ccdef3c4866c81 2024-11-10T09:17:13,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/d0e9d612008e41d695d57302cc312c97 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/d0e9d612008e41d695d57302cc312c97 2024-11-10T09:17:13,717 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/d0e9d612008e41d695d57302cc312c97, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T09:17:13,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/0ee55efd98214997b84ef388ef676cc1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/0ee55efd98214997b84ef388ef676cc1 2024-11-10T09:17:13,723 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/0ee55efd98214997b84ef388ef676cc1, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T09:17:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/300d024255af45d686ccdef3c4866c81 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/300d024255af45d686ccdef3c4866c81 2024-11-10T09:17:13,729 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/300d024255af45d686ccdef3c4866c81, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T09:17:13,730 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b031bdc33b2db2f1f99a0ca67fb65c56 in 948ms, sequenceid=69, compaction requested=false 2024-11-10T09:17:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-10T09:17:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-10T09:17:13,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T09:17:13,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2650 sec 2024-11-10T09:17:13,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.2770 sec 2024-11-10T09:17:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:13,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T09:17:13,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:13,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:13,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:13,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/ee6c472de0394a88b13036d470c1119d is 50, key is test_row_0/A:col10/1731230233520/Put/seqid=0 2024-11-10T09:17:13,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742087_1263 (size=12001) 2024-11-10T09:17:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230293869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230293871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230293977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230293977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230294187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230294187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:14,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/ee6c472de0394a88b13036d470c1119d 2024-11-10T09:17:14,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/2a5dd3f67a55468793e0204829ece1f6 is 50, key is test_row_0/B:col10/1731230233520/Put/seqid=0 2024-11-10T09:17:14,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742088_1264 (size=12001) 2024-11-10T09:17:14,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/2a5dd3f67a55468793e0204829ece1f6 2024-11-10T09:17:14,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/da1d2349d66b4f8784d7e68d43304aed is 50, key is test_row_0/C:col10/1731230233520/Put/seqid=0 2024-11-10T09:17:14,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742089_1265 (size=12001) 2024-11-10T09:17:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230294497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230294497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T09:17:14,627 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:14,627 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T09:17:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:17:14,631 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:14,632 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:14,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:14,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/da1d2349d66b4f8784d7e68d43304aed 2024-11-10T09:17:14,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/ee6c472de0394a88b13036d470c1119d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ee6c472de0394a88b13036d470c1119d 2024-11-10T09:17:14,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ee6c472de0394a88b13036d470c1119d, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T09:17:14,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/2a5dd3f67a55468793e0204829ece1f6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2a5dd3f67a55468793e0204829ece1f6 2024-11-10T09:17:14,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2a5dd3f67a55468793e0204829ece1f6, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T09:17:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/da1d2349d66b4f8784d7e68d43304aed as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/da1d2349d66b4f8784d7e68d43304aed 2024-11-10T09:17:14,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/da1d2349d66b4f8784d7e68d43304aed, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T09:17:14,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for b031bdc33b2db2f1f99a0ca67fb65c56 in 867ms, sequenceid=85, compaction requested=true 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:14,707 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:14,707 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:14,708 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:14,708 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:14,708 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:14,708 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:14,708 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:14,708 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/98de9732995543c48182531010212b21, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/d0e9d612008e41d695d57302cc312c97, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ee6c472de0394a88b13036d470c1119d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.3 K 2024-11-10T09:17:14,708 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:14,708 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/d87c38488db24936978ff5647a8259c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/0ee55efd98214997b84ef388ef676cc1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2a5dd3f67a55468793e0204829ece1f6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.3 K 2024-11-10T09:17:14,709 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d87c38488db24936978ff5647a8259c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230231194 2024-11-10T09:17:14,709 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98de9732995543c48182531010212b21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230231194 2024-11-10T09:17:14,709 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee55efd98214997b84ef388ef676cc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731230231333 2024-11-10T09:17:14,709 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e9d612008e41d695d57302cc312c97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731230231333 2024-11-10T09:17:14,710 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5dd3f67a55468793e0204829ece1f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731230233515 2024-11-10T09:17:14,710 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6c472de0394a88b13036d470c1119d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731230233515 2024-11-10T09:17:14,719 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:14,720 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4a2d55e5d79d4f679cf7a58a8c0125e4 is 50, key is test_row_0/B:col10/1731230233520/Put/seqid=0 2024-11-10T09:17:14,722 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:14,723 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8fa38dfbc0034856bbaa5759c66bea00 is 50, key is test_row_0/A:col10/1731230233520/Put/seqid=0 2024-11-10T09:17:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:17:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742091_1267 (size=12207) 2024-11-10T09:17:14,751 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8fa38dfbc0034856bbaa5759c66bea00 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8fa38dfbc0034856bbaa5759c66bea00 2024-11-10T09:17:14,758 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into 8fa38dfbc0034856bbaa5759c66bea00(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:14,758 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:14,758 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230234707; duration=0sec 2024-11-10T09:17:14,758 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:14,758 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:14,758 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:14,759 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:14,760 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:14,760 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:14,760 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/96194392d1f1430c86bdf432d5a32691, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/300d024255af45d686ccdef3c4866c81, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/da1d2349d66b4f8784d7e68d43304aed] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.3 K 2024-11-10T09:17:14,761 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96194392d1f1430c86bdf432d5a32691, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230231194 2024-11-10T09:17:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742090_1266 (size=12207) 2024-11-10T09:17:14,765 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 300d024255af45d686ccdef3c4866c81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731230231333 2024-11-10T09:17:14,766 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1d2349d66b4f8784d7e68d43304aed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731230233515 2024-11-10T09:17:14,769 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4a2d55e5d79d4f679cf7a58a8c0125e4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4a2d55e5d79d4f679cf7a58a8c0125e4 2024-11-10T09:17:14,775 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into 4a2d55e5d79d4f679cf7a58a8c0125e4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:14,775 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:14,775 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230234707; duration=0sec 2024-11-10T09:17:14,775 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:14,775 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:14,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T09:17:14,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:14,784 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:17:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:14,799 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:14,799 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4fbf4bc9ab994efba74f002e48a9b4f7 is 50, key is test_row_0/C:col10/1731230233520/Put/seqid=0 2024-11-10T09:17:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/6b60e22375a9473fa1a6ae3fc4e619be is 50, key is test_row_0/A:col10/1731230233868/Put/seqid=0 2024-11-10T09:17:14,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742092_1268 (size=9657) 2024-11-10T09:17:14,808 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/6b60e22375a9473fa1a6ae3fc4e619be 2024-11-10T09:17:14,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/31cee670beaf4f26a785f6a26e66a7aa is 50, key is test_row_0/B:col10/1731230233868/Put/seqid=0 2024-11-10T09:17:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742093_1269 (size=12207) 2024-11-10T09:17:14,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742094_1270 (size=9657) 2024-11-10T09:17:14,838 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/31cee670beaf4f26a785f6a26e66a7aa 2024-11-10T09:17:14,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/28300e6984a84443beb0164832613a9c is 50, key is test_row_0/C:col10/1731230233868/Put/seqid=0 2024-11-10T09:17:14,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742095_1271 (size=9657) 2024-11-10T09:17:14,870 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/28300e6984a84443beb0164832613a9c 2024-11-10T09:17:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/6b60e22375a9473fa1a6ae3fc4e619be as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6b60e22375a9473fa1a6ae3fc4e619be 2024-11-10T09:17:14,889 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6b60e22375a9473fa1a6ae3fc4e619be, entries=100, sequenceid=105, filesize=9.4 K 2024-11-10T09:17:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/31cee670beaf4f26a785f6a26e66a7aa as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/31cee670beaf4f26a785f6a26e66a7aa 2024-11-10T09:17:14,901 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/31cee670beaf4f26a785f6a26e66a7aa, entries=100, sequenceid=105, filesize=9.4 K 2024-11-10T09:17:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/28300e6984a84443beb0164832613a9c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/28300e6984a84443beb0164832613a9c 2024-11-10T09:17:14,914 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/28300e6984a84443beb0164832613a9c, entries=100, sequenceid=105, filesize=9.4 K 2024-11-10T09:17:14,916 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for b031bdc33b2db2f1f99a0ca67fb65c56 in 132ms, sequenceid=105, compaction requested=false 2024-11-10T09:17:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T09:17:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T09:17:14,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T09:17:14,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 288 msec 2024-11-10T09:17:14,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 299 msec 2024-11-10T09:17:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:17:14,948 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:14,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T09:17:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:17:14,952 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:14,953 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:14,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:15,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:17:15,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:15,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:15,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:15,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:15,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:15,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:15,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/bf3bd64f718c49fc86d7ae2253133287 is 50, key is test_row_0/A:col10/1731230235014/Put/seqid=0 2024-11-10T09:17:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742096_1272 (size=14341) 2024-11-10T09:17:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:17:15,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230235056, max timestamp=1731230235077Num uniques 859; 2024-11-10T09:17:15,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230235056, max timestamp=1731230235077Num uniques 859; 2024-11-10T09:17:15,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:15,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:15,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230235056, max timestamp=1731230235077Num uniques 859; 2024-11-10T09:17:15,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:15,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230295115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230295121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,225 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4fbf4bc9ab994efba74f002e48a9b4f7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4fbf4bc9ab994efba74f002e48a9b4f7 2024-11-10T09:17:15,229 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into 4fbf4bc9ab994efba74f002e48a9b4f7(size=11.9 K), total size for store is 21.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230295227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230295228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,232 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:15,232 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230234707; duration=0sec 2024-11-10T09:17:15,232 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:15,232 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:15,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:17:15,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230295437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230295437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/bf3bd64f718c49fc86d7ae2253133287 2024-11-10T09:17:15,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4cd540fb564d4b91bf3a3bdc885fb384 is 50, key is test_row_0/B:col10/1731230235014/Put/seqid=0 2024-11-10T09:17:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742097_1273 (size=12001) 2024-11-10T09:17:15,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230295537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,543 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230295537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,545 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230295548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,554 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:15,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:15,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:17:15,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:15,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230295747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230295747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:15,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:15,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4cd540fb564d4b91bf3a3bdc885fb384 2024-11-10T09:17:15,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8166e540441148cf911914ae553d680f is 50, key is test_row_0/C:col10/1731230235014/Put/seqid=0 2024-11-10T09:17:15,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742098_1274 (size=12001) 2024-11-10T09:17:15,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8166e540441148cf911914ae553d680f 2024-11-10T09:17:15,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/bf3bd64f718c49fc86d7ae2253133287 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/bf3bd64f718c49fc86d7ae2253133287 2024-11-10T09:17:15,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/bf3bd64f718c49fc86d7ae2253133287, entries=200, sequenceid=116, filesize=14.0 K 2024-11-10T09:17:15,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4cd540fb564d4b91bf3a3bdc885fb384 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4cd540fb564d4b91bf3a3bdc885fb384 2024-11-10T09:17:15,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4cd540fb564d4b91bf3a3bdc885fb384, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T09:17:15,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8166e540441148cf911914ae553d680f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8166e540441148cf911914ae553d680f 2024-11-10T09:17:15,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8166e540441148cf911914ae553d680f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T09:17:15,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b031bdc33b2db2f1f99a0ca67fb65c56 in 936ms, sequenceid=116, compaction requested=true 2024-11-10T09:17:15,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:15,953 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:15,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:15,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:15,955 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:15,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:15,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:15,955 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,956 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8fa38dfbc0034856bbaa5759c66bea00, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6b60e22375a9473fa1a6ae3fc4e619be, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/bf3bd64f718c49fc86d7ae2253133287] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.4 K 2024-11-10T09:17:15,956 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa38dfbc0034856bbaa5759c66bea00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731230233515 2024-11-10T09:17:15,956 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b60e22375a9473fa1a6ae3fc4e619be, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230233864 2024-11-10T09:17:15,957 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf3bd64f718c49fc86d7ae2253133287, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230235010 2024-11-10T09:17:15,957 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:15,957 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:15,957 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:15,957 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4a2d55e5d79d4f679cf7a58a8c0125e4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/31cee670beaf4f26a785f6a26e66a7aa, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4cd540fb564d4b91bf3a3bdc885fb384] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=33.1 K 2024-11-10T09:17:15,958 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a2d55e5d79d4f679cf7a58a8c0125e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731230233515 2024-11-10T09:17:15,958 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cee670beaf4f26a785f6a26e66a7aa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230233864 2024-11-10T09:17:15,958 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd540fb564d4b91bf3a3bdc885fb384, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230235014 2024-11-10T09:17:15,975 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:15,976 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/7da9ee7e918d47e39a5bf1301e4bb3be is 50, key is test_row_0/A:col10/1731230235014/Put/seqid=0 2024-11-10T09:17:15,988 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:15,989 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/e55dbeb9cff74337b0b1aab518670a2d is 50, key is test_row_0/B:col10/1731230235014/Put/seqid=0 2024-11-10T09:17:16,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742099_1275 (size=12309) 2024-11-10T09:17:16,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742100_1276 (size=12309) 2024-11-10T09:17:16,012 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/e55dbeb9cff74337b0b1aab518670a2d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e55dbeb9cff74337b0b1aab518670a2d 2024-11-10T09:17:16,024 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into e55dbeb9cff74337b0b1aab518670a2d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:16,024 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:16,024 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230235955; duration=0sec 2024-11-10T09:17:16,024 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:16,024 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:16,024 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:16,025 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:16,025 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:16,025 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:16,026 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4fbf4bc9ab994efba74f002e48a9b4f7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/28300e6984a84443beb0164832613a9c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8166e540441148cf911914ae553d680f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=33.1 K 2024-11-10T09:17:16,026 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fbf4bc9ab994efba74f002e48a9b4f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731230233515 2024-11-10T09:17:16,026 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 28300e6984a84443beb0164832613a9c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731230233864 2024-11-10T09:17:16,026 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8166e540441148cf911914ae553d680f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230235014 2024-11-10T09:17:16,032 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:16,033 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c818e7e2b0fa4781ba8383d4be1b9b87 is 50, key is test_row_0/C:col10/1731230235014/Put/seqid=0 2024-11-10T09:17:16,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742101_1277 (size=12309) 2024-11-10T09:17:16,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:17:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:16,042 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T09:17:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:16,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/062dabcff88b45fd88a70541a26f539d is 50, key is test_row_0/A:col10/1731230235119/Put/seqid=0 2024-11-10T09:17:16,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742102_1278 (size=12151) 2024-11-10T09:17:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:17:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:16,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230296287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230296288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230296397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230296398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,411 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/7da9ee7e918d47e39a5bf1301e4bb3be as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/7da9ee7e918d47e39a5bf1301e4bb3be 2024-11-10T09:17:16,416 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into 7da9ee7e918d47e39a5bf1301e4bb3be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:16,416 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:16,416 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230235953; duration=0sec 2024-11-10T09:17:16,416 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:16,416 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:16,452 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/062dabcff88b45fd88a70541a26f539d 2024-11-10T09:17:16,457 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c818e7e2b0fa4781ba8383d4be1b9b87 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c818e7e2b0fa4781ba8383d4be1b9b87 2024-11-10T09:17:16,470 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into c818e7e2b0fa4781ba8383d4be1b9b87(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:16,471 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:16,471 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230235955; duration=0sec 2024-11-10T09:17:16,471 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:16,471 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:16,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/fa57f9242ec94aefa32a2882705c19f8 is 50, key is test_row_0/B:col10/1731230235119/Put/seqid=0 2024-11-10T09:17:16,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742103_1279 (size=12151) 2024-11-10T09:17:16,526 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/fa57f9242ec94aefa32a2882705c19f8 2024-11-10T09:17:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c5f3937a46e54d058c4ef2e2bb36c1df is 50, key is test_row_0/C:col10/1731230235119/Put/seqid=0 2024-11-10T09:17:16,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742104_1280 (size=12151) 2024-11-10T09:17:16,607 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c5f3937a46e54d058c4ef2e2bb36c1df 2024-11-10T09:17:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230296607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230296608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/062dabcff88b45fd88a70541a26f539d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/062dabcff88b45fd88a70541a26f539d 2024-11-10T09:17:16,628 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/062dabcff88b45fd88a70541a26f539d, entries=150, sequenceid=153, filesize=11.9 K 2024-11-10T09:17:16,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/fa57f9242ec94aefa32a2882705c19f8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/fa57f9242ec94aefa32a2882705c19f8 2024-11-10T09:17:16,638 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/fa57f9242ec94aefa32a2882705c19f8, entries=150, sequenceid=153, filesize=11.9 K 2024-11-10T09:17:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c5f3937a46e54d058c4ef2e2bb36c1df as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c5f3937a46e54d058c4ef2e2bb36c1df 2024-11-10T09:17:16,662 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c5f3937a46e54d058c4ef2e2bb36c1df, entries=150, sequenceid=153, filesize=11.9 K 2024-11-10T09:17:16,663 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for b031bdc33b2db2f1f99a0ca67fb65c56 in 621ms, sequenceid=153, compaction requested=false 2024-11-10T09:17:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T09:17:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T09:17:16,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T09:17:16,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-11-10T09:17:16,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7200 sec 2024-11-10T09:17:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:16,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:17:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:16,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/db29550a111f454b8ebb5ec60c0e453e is 50, key is test_row_0/A:col10/1731230236278/Put/seqid=0 2024-11-10T09:17:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742105_1281 (size=14541) 2024-11-10T09:17:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230296978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230296978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230297087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:17,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230297087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:17:17,107 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:17,107 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T09:17:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:17:17,109 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:17,110 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:17,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:17:17,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T09:17:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230297297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:17,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230297297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/db29550a111f454b8ebb5ec60c0e453e 2024-11-10T09:17:17,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/65d5dd4f19dc40d69199d0df2a868467 is 50, key is test_row_0/B:col10/1731230236278/Put/seqid=0 2024-11-10T09:17:17,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742106_1282 (size=12151) 2024-11-10T09:17:17,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T09:17:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:17:17,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T09:17:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230297613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230297613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:17,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T09:17:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:17:17,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/65d5dd4f19dc40d69199d0df2a868467 2024-11-10T09:17:17,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/3cf1a7c59eaf47e98e10a347c840f211 is 50, key is test_row_0/C:col10/1731230236278/Put/seqid=0 2024-11-10T09:17:17,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742107_1283 (size=12151) 2024-11-10T09:17:17,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/3cf1a7c59eaf47e98e10a347c840f211 2024-11-10T09:17:17,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/db29550a111f454b8ebb5ec60c0e453e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/db29550a111f454b8ebb5ec60c0e453e 2024-11-10T09:17:17,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/db29550a111f454b8ebb5ec60c0e453e, entries=200, sequenceid=167, filesize=14.2 K 2024-11-10T09:17:17,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/65d5dd4f19dc40d69199d0df2a868467 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/65d5dd4f19dc40d69199d0df2a868467 2024-11-10T09:17:17,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/65d5dd4f19dc40d69199d0df2a868467, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T09:17:17,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/3cf1a7c59eaf47e98e10a347c840f211 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/3cf1a7c59eaf47e98e10a347c840f211 2024-11-10T09:17:17,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/3cf1a7c59eaf47e98e10a347c840f211, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T09:17:17,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b031bdc33b2db2f1f99a0ca67fb65c56 in 944ms, sequenceid=167, compaction requested=true 2024-11-10T09:17:17,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:17,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:17,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:17,864 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:17,864 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:17,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:17,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:17,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:17,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:17,866 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:17,866 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:17,866 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:17,866 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:17,866 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,866 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,866 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/7da9ee7e918d47e39a5bf1301e4bb3be, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/062dabcff88b45fd88a70541a26f539d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/db29550a111f454b8ebb5ec60c0e453e] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=38.1 K 2024-11-10T09:17:17,867 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e55dbeb9cff74337b0b1aab518670a2d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/fa57f9242ec94aefa32a2882705c19f8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/65d5dd4f19dc40d69199d0df2a868467] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.8 K 2024-11-10T09:17:17,867 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e55dbeb9cff74337b0b1aab518670a2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230235014 2024-11-10T09:17:17,867 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da9ee7e918d47e39a5bf1301e4bb3be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230235014 2024-11-10T09:17:17,867 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting fa57f9242ec94aefa32a2882705c19f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731230235111 2024-11-10T09:17:17,867 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062dabcff88b45fd88a70541a26f539d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731230235111 2024-11-10T09:17:17,868 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d5dd4f19dc40d69199d0df2a868467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731230236278 2024-11-10T09:17:17,868 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting db29550a111f454b8ebb5ec60c0e453e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731230236278 2024-11-10T09:17:17,882 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:17,882 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/7f44d043f9c443fb9eb4fdf678295830 is 50, key is test_row_0/B:col10/1731230236278/Put/seqid=0 2024-11-10T09:17:17,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:17,888 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:17,894 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:17,895 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/fbe2c920acad45c2a86dfc92dd50d9b1 is 50, key is test_row_0/A:col10/1731230236278/Put/seqid=0 2024-11-10T09:17:17,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/50dfe81850bb4b6ca57883b7b9a7017f is 50, key is test_row_0/A:col10/1731230236977/Put/seqid=0 2024-11-10T09:17:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742108_1284 (size=12561) 2024-11-10T09:17:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742109_1285 (size=12561) 2024-11-10T09:17:17,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742110_1286 (size=12151) 2024-11-10T09:17:17,997 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/fbe2c920acad45c2a86dfc92dd50d9b1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/fbe2c920acad45c2a86dfc92dd50d9b1 2024-11-10T09:17:18,009 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into fbe2c920acad45c2a86dfc92dd50d9b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:18,009 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:18,009 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230237864; duration=0sec 2024-11-10T09:17:18,010 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:18,010 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:18,010 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:18,018 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:18,018 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:18,018 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:18,019 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c818e7e2b0fa4781ba8383d4be1b9b87, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c5f3937a46e54d058c4ef2e2bb36c1df, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/3cf1a7c59eaf47e98e10a347c840f211] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=35.8 K 2024-11-10T09:17:18,021 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c818e7e2b0fa4781ba8383d4be1b9b87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230235014 2024-11-10T09:17:18,025 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f3937a46e54d058c4ef2e2bb36c1df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731230235111 2024-11-10T09:17:18,025 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cf1a7c59eaf47e98e10a347c840f211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731230236278 2024-11-10T09:17:18,051 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:18,052 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/b48419d6b3244a3891db45bb094039a7 is 50, key is test_row_0/C:col10/1731230236278/Put/seqid=0 2024-11-10T09:17:18,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742111_1287 (size=12561) 2024-11-10T09:17:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:18,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:17:18,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T09:17:18,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230298161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230298167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:17:18,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230298279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230298280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,385 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/7f44d043f9c443fb9eb4fdf678295830 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/7f44d043f9c443fb9eb4fdf678295830 2024-11-10T09:17:18,390 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/50dfe81850bb4b6ca57883b7b9a7017f 2024-11-10T09:17:18,399 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into 7f44d043f9c443fb9eb4fdf678295830(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:18,399 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:18,399 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230237864; duration=0sec 2024-11-10T09:17:18,399 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:18,399 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:18,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/ef9192d2c6f6498b97d5cee619a07260 is 50, key is test_row_0/B:col10/1731230236977/Put/seqid=0 2024-11-10T09:17:18,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742112_1288 (size=12151) 2024-11-10T09:17:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230298487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230298499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,536 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/b48419d6b3244a3891db45bb094039a7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/b48419d6b3244a3891db45bb094039a7 2024-11-10T09:17:18,577 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into b48419d6b3244a3891db45bb094039a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:18,577 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:18,577 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230237865; duration=0sec 2024-11-10T09:17:18,577 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:18,577 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:18,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230298797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230298820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:18,830 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/ef9192d2c6f6498b97d5cee619a07260 2024-11-10T09:17:18,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/bc05b45141b447bd9e8035da608c6329 is 50, key is test_row_0/C:col10/1731230236977/Put/seqid=0 2024-11-10T09:17:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742113_1289 (size=12151) 2024-11-10T09:17:18,903 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/bc05b45141b447bd9e8035da608c6329 2024-11-10T09:17:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/50dfe81850bb4b6ca57883b7b9a7017f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/50dfe81850bb4b6ca57883b7b9a7017f 2024-11-10T09:17:18,935 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/50dfe81850bb4b6ca57883b7b9a7017f, entries=150, sequenceid=187, filesize=11.9 K 2024-11-10T09:17:18,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/ef9192d2c6f6498b97d5cee619a07260 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ef9192d2c6f6498b97d5cee619a07260 2024-11-10T09:17:18,944 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ef9192d2c6f6498b97d5cee619a07260, entries=150, sequenceid=187, filesize=11.9 K 2024-11-10T09:17:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/bc05b45141b447bd9e8035da608c6329 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/bc05b45141b447bd9e8035da608c6329 2024-11-10T09:17:18,959 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/bc05b45141b447bd9e8035da608c6329, entries=150, sequenceid=187, filesize=11.9 K 2024-11-10T09:17:18,961 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1073ms, sequenceid=187, compaction requested=false 2024-11-10T09:17:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T09:17:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T09:17:18,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T09:17:18,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-10T09:17:19,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8940 sec 2024-11-10T09:17:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:17:19,257 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:19,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T09:17:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:17:19,269 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:19,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:19,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:19,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:17:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:19,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/93fdfb35bf824c15b601c120dcf72d00 is 50, key is test_row_0/A:col10/1731230238154/Put/seqid=0 2024-11-10T09:17:19,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742114_1290 (size=12151) 2024-11-10T09:17:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:17:19,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230299396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230299406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:17:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230299517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230299518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230299562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,567 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:17:19,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:17:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230299588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,597 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230299597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,601 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230299727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230299728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:19,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:17:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/93fdfb35bf824c15b601c120dcf72d00 2024-11-10T09:17:19,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b145ef70452d4d5caafdba6789ddd3a2 is 50, key is test_row_0/B:col10/1731230238154/Put/seqid=0 2024-11-10T09:17:19,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742115_1291 (size=12151) 2024-11-10T09:17:19,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b145ef70452d4d5caafdba6789ddd3a2 2024-11-10T09:17:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:17:19,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:17:19,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:19,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:19,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:19,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/e53943c948194fb48c45ccdf221064d7 is 50, key is test_row_0/C:col10/1731230238154/Put/seqid=0 2024-11-10T09:17:19,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742116_1292 (size=12151) 2024-11-10T09:17:19,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/e53943c948194fb48c45ccdf221064d7 2024-11-10T09:17:20,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230300037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230300048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:20,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:17:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:20,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/93fdfb35bf824c15b601c120dcf72d00 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/93fdfb35bf824c15b601c120dcf72d00 2024-11-10T09:17:20,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/93fdfb35bf824c15b601c120dcf72d00, entries=150, sequenceid=203, filesize=11.9 K 2024-11-10T09:17:20,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b145ef70452d4d5caafdba6789ddd3a2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b145ef70452d4d5caafdba6789ddd3a2 2024-11-10T09:17:20,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b145ef70452d4d5caafdba6789ddd3a2, entries=150, sequenceid=203, filesize=11.9 K 2024-11-10T09:17:20,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/e53943c948194fb48c45ccdf221064d7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/e53943c948194fb48c45ccdf221064d7 2024-11-10T09:17:20,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/e53943c948194fb48c45ccdf221064d7, entries=150, sequenceid=203, filesize=11.9 K 2024-11-10T09:17:20,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b031bdc33b2db2f1f99a0ca67fb65c56 in 878ms, sequenceid=203, compaction requested=true 2024-11-10T09:17:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:20,189 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:20,189 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:20,194 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:20,194 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:20,194 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:20,194 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,194 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:20,194 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,194 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/7f44d043f9c443fb9eb4fdf678295830, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ef9192d2c6f6498b97d5cee619a07260, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b145ef70452d4d5caafdba6789ddd3a2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.0 K 2024-11-10T09:17:20,194 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/fbe2c920acad45c2a86dfc92dd50d9b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/50dfe81850bb4b6ca57883b7b9a7017f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/93fdfb35bf824c15b601c120dcf72d00] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.0 K 2024-11-10T09:17:20,195 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe2c920acad45c2a86dfc92dd50d9b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731230236278 2024-11-10T09:17:20,196 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f44d043f9c443fb9eb4fdf678295830, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731230236278 2024-11-10T09:17:20,196 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50dfe81850bb4b6ca57883b7b9a7017f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731230236972 2024-11-10T09:17:20,197 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93fdfb35bf824c15b601c120dcf72d00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230238154 2024-11-10T09:17:20,198 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ef9192d2c6f6498b97d5cee619a07260, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731230236972 2024-11-10T09:17:20,198 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b145ef70452d4d5caafdba6789ddd3a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230238154 2024-11-10T09:17:20,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:17:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,220 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T09:17:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:20,227 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:20,227 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/df88a725a6cf49caae8b2b517a8fb449 is 50, key is test_row_0/A:col10/1731230238154/Put/seqid=0 2024-11-10T09:17:20,234 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:20,235 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/c232a0fbbd8e49108df4b2604b242b8c is 50, key is test_row_0/B:col10/1731230238154/Put/seqid=0 2024-11-10T09:17:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8517829f4ac74e908b75baa52ecf680b is 50, key is test_row_0/A:col10/1731230239394/Put/seqid=0 2024-11-10T09:17:20,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742118_1294 (size=12663) 2024-11-10T09:17:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742117_1293 (size=12663) 2024-11-10T09:17:20,296 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/df88a725a6cf49caae8b2b517a8fb449 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/df88a725a6cf49caae8b2b517a8fb449 2024-11-10T09:17:20,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742119_1295 (size=12151) 2024-11-10T09:17:20,302 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8517829f4ac74e908b75baa52ecf680b 2024-11-10T09:17:20,322 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into df88a725a6cf49caae8b2b517a8fb449(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:20,322 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:20,323 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230240189; duration=0sec 2024-11-10T09:17:20,323 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:20,323 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:20,324 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:20,334 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:20,335 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:20,335 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,335 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/b48419d6b3244a3891db45bb094039a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/bc05b45141b447bd9e8035da608c6329, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/e53943c948194fb48c45ccdf221064d7] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.0 K 2024-11-10T09:17:20,340 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48419d6b3244a3891db45bb094039a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731230236278 2024-11-10T09:17:20,344 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc05b45141b447bd9e8035da608c6329, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731230236972 2024-11-10T09:17:20,349 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53943c948194fb48c45ccdf221064d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230238154 2024-11-10T09:17:20,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4b4b520f918d4918aa214d46bc5fd885 is 50, key is test_row_0/B:col10/1731230239394/Put/seqid=0 2024-11-10T09:17:20,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742120_1296 (size=12151) 2024-11-10T09:17:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:17:20,418 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:20,419 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/16b5ad76f8dd455f80f853779af6c613 is 50, key is test_row_0/C:col10/1731230238154/Put/seqid=0 2024-11-10T09:17:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742121_1297 (size=12663) 2024-11-10T09:17:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:20,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230300664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:20,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230300665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:20,709 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/c232a0fbbd8e49108df4b2604b242b8c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/c232a0fbbd8e49108df4b2604b242b8c 2024-11-10T09:17:20,720 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into c232a0fbbd8e49108df4b2604b242b8c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:20,720 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:20,720 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230240189; duration=0sec 2024-11-10T09:17:20,721 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:20,721 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230300777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230300777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:20,810 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4b4b520f918d4918aa214d46bc5fd885 2024-11-10T09:17:20,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7929961c401d4de099430d62c3b83512 is 50, key is test_row_0/C:col10/1731230239394/Put/seqid=0 2024-11-10T09:17:20,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742122_1298 (size=12151) 2024-11-10T09:17:20,850 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7929961c401d4de099430d62c3b83512 2024-11-10T09:17:20,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/8517829f4ac74e908b75baa52ecf680b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8517829f4ac74e908b75baa52ecf680b 2024-11-10T09:17:20,861 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8517829f4ac74e908b75baa52ecf680b, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T09:17:20,862 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/16b5ad76f8dd455f80f853779af6c613 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/16b5ad76f8dd455f80f853779af6c613 2024-11-10T09:17:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/4b4b520f918d4918aa214d46bc5fd885 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4b4b520f918d4918aa214d46bc5fd885 2024-11-10T09:17:20,866 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4b4b520f918d4918aa214d46bc5fd885, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T09:17:20,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7929961c401d4de099430d62c3b83512 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7929961c401d4de099430d62c3b83512 2024-11-10T09:17:20,867 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into 16b5ad76f8dd455f80f853779af6c613(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:20,867 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:20,867 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230240191; duration=0sec 2024-11-10T09:17:20,867 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:20,868 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:20,871 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7929961c401d4de099430d62c3b83512, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T09:17:20,872 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b031bdc33b2db2f1f99a0ca67fb65c56 in 653ms, sequenceid=222, compaction requested=false 2024-11-10T09:17:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T09:17:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T09:17:20,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T09:17:20,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6020 sec 2024-11-10T09:17:20,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6150 sec 2024-11-10T09:17:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:20,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:17:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:20,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/b4b601b96fd144fc837cc7239064c135 is 50, key is test_row_0/A:col10/1731230240988/Put/seqid=0 2024-11-10T09:17:21,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742123_1299 (size=14541) 2024-11-10T09:17:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230301021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230301026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230301137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230301137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230301347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230301347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/b4b601b96fd144fc837cc7239064c135 2024-11-10T09:17:21,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/e5a3fa590f6641a38bcc64106a45462d is 50, key is test_row_0/B:col10/1731230240988/Put/seqid=0 2024-11-10T09:17:21,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742124_1300 (size=12151) 2024-11-10T09:17:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:17:21,427 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:21,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T09:17:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:17:21,429 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:21,430 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:21,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:17:21,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:21,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230301657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230301657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:21,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:17:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:21,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:17:21,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/e5a3fa590f6641a38bcc64106a45462d 2024-11-10T09:17:21,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/6ab846a13f8f41dbb46e5820be294bb2 is 50, key is test_row_0/C:col10/1731230240988/Put/seqid=0 2024-11-10T09:17:21,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:17:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:21,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:21,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742125_1301 (size=12151) 2024-11-10T09:17:22,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:17:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:17:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230302167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230302167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:22,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:17:22,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:22,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:22,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:22,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/6ab846a13f8f41dbb46e5820be294bb2 2024-11-10T09:17:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/b4b601b96fd144fc837cc7239064c135 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/b4b601b96fd144fc837cc7239064c135 2024-11-10T09:17:22,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/b4b601b96fd144fc837cc7239064c135, entries=200, sequenceid=240, filesize=14.2 K 2024-11-10T09:17:22,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/e5a3fa590f6641a38bcc64106a45462d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e5a3fa590f6641a38bcc64106a45462d 2024-11-10T09:17:22,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e5a3fa590f6641a38bcc64106a45462d, entries=150, sequenceid=240, filesize=11.9 K 2024-11-10T09:17:22,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/6ab846a13f8f41dbb46e5820be294bb2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/6ab846a13f8f41dbb46e5820be294bb2 2024-11-10T09:17:22,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/6ab846a13f8f41dbb46e5820be294bb2, entries=150, sequenceid=240, filesize=11.9 K 2024-11-10T09:17:22,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1328ms, sequenceid=240, compaction requested=true 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:22,318 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:22,318 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:22,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:22,319 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:22,319 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:22,319 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:22,319 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:22,319 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,319 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/c232a0fbbd8e49108df4b2604b242b8c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4b4b520f918d4918aa214d46bc5fd885, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e5a3fa590f6641a38bcc64106a45462d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.1 K 2024-11-10T09:17:22,319 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,319 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/df88a725a6cf49caae8b2b517a8fb449, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8517829f4ac74e908b75baa52ecf680b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/b4b601b96fd144fc837cc7239064c135] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=38.4 K 2024-11-10T09:17:22,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c232a0fbbd8e49108df4b2604b242b8c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230238154 2024-11-10T09:17:22,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting df88a725a6cf49caae8b2b517a8fb449, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230238154 2024-11-10T09:17:22,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4b520f918d4918aa214d46bc5fd885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731230239386 2024-11-10T09:17:22,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8517829f4ac74e908b75baa52ecf680b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731230239386 2024-11-10T09:17:22,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a3fa590f6641a38bcc64106a45462d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731230240576 2024-11-10T09:17:22,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4b601b96fd144fc837cc7239064c135, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731230240576 2024-11-10T09:17:22,338 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:22,338 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/c22031829420425baa10941cb41fe2e3 is 50, key is test_row_0/A:col10/1731230240988/Put/seqid=0 2024-11-10T09:17:22,338 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:22,339 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/beadd40876e14850a6bfc58f76711aee is 50, key is test_row_0/B:col10/1731230240988/Put/seqid=0 2024-11-10T09:17:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742127_1303 (size=12765) 2024-11-10T09:17:22,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742126_1302 (size=12765) 2024-11-10T09:17:22,355 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/c22031829420425baa10941cb41fe2e3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/c22031829420425baa10941cb41fe2e3 2024-11-10T09:17:22,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,358 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:22,361 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into c22031829420425baa10941cb41fe2e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:22,361 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:22,361 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230242318; duration=0sec 2024-11-10T09:17:22,362 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:22,362 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:22,362 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:22,363 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:22,363 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:22,363 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:22,363 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/16b5ad76f8dd455f80f853779af6c613, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7929961c401d4de099430d62c3b83512, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/6ab846a13f8f41dbb46e5820be294bb2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.1 K 2024-11-10T09:17:22,364 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b5ad76f8dd455f80f853779af6c613, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230238154 2024-11-10T09:17:22,364 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7929961c401d4de099430d62c3b83512, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731230239386 2024-11-10T09:17:22,364 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab846a13f8f41dbb46e5820be294bb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731230240576 2024-11-10T09:17:22,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/035c493c74134913a2c0e108e8e24484 is 50, key is test_row_0/A:col10/1731230241019/Put/seqid=0 2024-11-10T09:17:22,375 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:22,375 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/eb0f1c9a9f114e31a3e5018e10f1bcc1 is 50, key is test_row_0/C:col10/1731230240988/Put/seqid=0 2024-11-10T09:17:22,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742128_1304 (size=12151) 2024-11-10T09:17:22,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742129_1305 (size=12765) 2024-11-10T09:17:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:17:22,755 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/beadd40876e14850a6bfc58f76711aee as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/beadd40876e14850a6bfc58f76711aee 2024-11-10T09:17:22,760 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into beadd40876e14850a6bfc58f76711aee(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:22,760 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:22,760 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230242318; duration=0sec 2024-11-10T09:17:22,760 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:22,760 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:22,777 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/035c493c74134913a2c0e108e8e24484 2024-11-10T09:17:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b1ce6d63af59437da766fd39b8fd7043 is 50, key is test_row_0/B:col10/1731230241019/Put/seqid=0 2024-11-10T09:17:22,795 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/eb0f1c9a9f114e31a3e5018e10f1bcc1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/eb0f1c9a9f114e31a3e5018e10f1bcc1 2024-11-10T09:17:22,803 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into eb0f1c9a9f114e31a3e5018e10f1bcc1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:22,803 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:22,803 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230242318; duration=0sec 2024-11-10T09:17:22,803 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:22,803 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:22,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742130_1306 (size=12151) 2024-11-10T09:17:22,806 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b1ce6d63af59437da766fd39b8fd7043 2024-11-10T09:17:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/994da08b25124ffd8799c65e2d2ea0e1 is 50, key is test_row_0/C:col10/1731230241019/Put/seqid=0 2024-11-10T09:17:22,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742131_1307 (size=12151) 2024-11-10T09:17:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:23,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:23,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230303217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230303217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,245 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/994da08b25124ffd8799c65e2d2ea0e1 2024-11-10T09:17:23,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/035c493c74134913a2c0e108e8e24484 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/035c493c74134913a2c0e108e8e24484 2024-11-10T09:17:23,254 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/035c493c74134913a2c0e108e8e24484, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T09:17:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b1ce6d63af59437da766fd39b8fd7043 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b1ce6d63af59437da766fd39b8fd7043 2024-11-10T09:17:23,258 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b1ce6d63af59437da766fd39b8fd7043, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T09:17:23,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/994da08b25124ffd8799c65e2d2ea0e1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/994da08b25124ffd8799c65e2d2ea0e1 2024-11-10T09:17:23,262 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/994da08b25124ffd8799c65e2d2ea0e1, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T09:17:23,263 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b031bdc33b2db2f1f99a0ca67fb65c56 in 905ms, sequenceid=257, compaction requested=false 2024-11-10T09:17:23,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:23,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:23,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T09:17:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T09:17:23,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T09:17:23,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-10T09:17:23,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.8380 sec 2024-11-10T09:17:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:23,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T09:17:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:23,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/6ccd7bbad4024b079bcdc9335191eab9 is 50, key is test_row_0/A:col10/1731230243329/Put/seqid=0 2024-11-10T09:17:23,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742132_1308 (size=14741) 2024-11-10T09:17:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230303389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230303390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230303498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230303498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:17:23,577 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:23,578 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T09:17:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:17:23,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:23,580 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:23,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:17:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230303707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230303707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:23,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:17:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:23,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:23,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:23,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/6ccd7bbad4024b079bcdc9335191eab9 2024-11-10T09:17:23,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/9c9a4e47ff444069b89bb896873beed6 is 50, key is test_row_0/B:col10/1731230243329/Put/seqid=0 2024-11-10T09:17:23,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742133_1309 (size=12301) 2024-11-10T09:17:23,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/9c9a4e47ff444069b89bb896873beed6 2024-11-10T09:17:23,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/88a1e2ad512746059133f9c8de74a6b5 is 50, key is test_row_0/C:col10/1731230243329/Put/seqid=0 2024-11-10T09:17:23,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742134_1310 (size=12301) 2024-11-10T09:17:23,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:23,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:17:24,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:24,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230304017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230304017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:24,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:17:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:24,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/88a1e2ad512746059133f9c8de74a6b5 2024-11-10T09:17:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/6ccd7bbad4024b079bcdc9335191eab9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6ccd7bbad4024b079bcdc9335191eab9 2024-11-10T09:17:24,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6ccd7bbad4024b079bcdc9335191eab9, entries=200, sequenceid=277, filesize=14.4 K 2024-11-10T09:17:24,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/9c9a4e47ff444069b89bb896873beed6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/9c9a4e47ff444069b89bb896873beed6 2024-11-10T09:17:24,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/9c9a4e47ff444069b89bb896873beed6, entries=150, sequenceid=277, filesize=12.0 K 2024-11-10T09:17:24,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/88a1e2ad512746059133f9c8de74a6b5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/88a1e2ad512746059133f9c8de74a6b5 2024-11-10T09:17:24,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/88a1e2ad512746059133f9c8de74a6b5, entries=150, sequenceid=277, filesize=12.0 K 2024-11-10T09:17:24,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b031bdc33b2db2f1f99a0ca67fb65c56 in 851ms, sequenceid=277, compaction requested=true 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:24,183 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:24,183 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:24,189 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:24,189 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:24,190 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,190 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/beadd40876e14850a6bfc58f76711aee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b1ce6d63af59437da766fd39b8fd7043, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/9c9a4e47ff444069b89bb896873beed6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.3 K 2024-11-10T09:17:24,190 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:24,190 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:24,190 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,190 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/c22031829420425baa10941cb41fe2e3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/035c493c74134913a2c0e108e8e24484, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6ccd7bbad4024b079bcdc9335191eab9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=38.7 K 2024-11-10T09:17:24,190 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting beadd40876e14850a6bfc58f76711aee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731230240576 2024-11-10T09:17:24,191 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22031829420425baa10941cb41fe2e3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731230240576 2024-11-10T09:17:24,191 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ce6d63af59437da766fd39b8fd7043, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230241008 2024-11-10T09:17:24,191 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035c493c74134913a2c0e108e8e24484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230241008 2024-11-10T09:17:24,191 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ccd7bbad4024b079bcdc9335191eab9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230243215 2024-11-10T09:17:24,191 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9a4e47ff444069b89bb896873beed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230243215 2024-11-10T09:17:24,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:17:24,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,192 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T09:17:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:24,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/dec8169b84a145a6966449e1f06db51b is 50, key is test_row_0/A:col10/1731230243388/Put/seqid=0 2024-11-10T09:17:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:17:24,218 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#265 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:24,218 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:24,219 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/11b6aaf8b9794300bd0392f7767b67b5 is 50, key is test_row_0/A:col10/1731230243329/Put/seqid=0 2024-11-10T09:17:24,219 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/bf3af1641acf496db8e72d1234927fa0 is 50, key is test_row_0/B:col10/1731230243329/Put/seqid=0 2024-11-10T09:17:24,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742135_1311 (size=12301) 2024-11-10T09:17:24,242 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/dec8169b84a145a6966449e1f06db51b 2024-11-10T09:17:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/72b0f40612ed4bf09922f5aa0cb139b4 is 50, key is test_row_0/B:col10/1731230243388/Put/seqid=0 2024-11-10T09:17:24,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742136_1312 (size=13017) 2024-11-10T09:17:24,295 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/11b6aaf8b9794300bd0392f7767b67b5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/11b6aaf8b9794300bd0392f7767b67b5 2024-11-10T09:17:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742137_1313 (size=13017) 2024-11-10T09:17:24,309 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/bf3af1641acf496db8e72d1234927fa0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/bf3af1641acf496db8e72d1234927fa0 2024-11-10T09:17:24,310 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into 11b6aaf8b9794300bd0392f7767b67b5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:24,310 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:24,310 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230244183; duration=0sec 2024-11-10T09:17:24,311 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:24,311 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:24,311 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:24,312 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:24,312 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:24,312 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,313 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/eb0f1c9a9f114e31a3e5018e10f1bcc1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/994da08b25124ffd8799c65e2d2ea0e1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/88a1e2ad512746059133f9c8de74a6b5] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.3 K 2024-11-10T09:17:24,313 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb0f1c9a9f114e31a3e5018e10f1bcc1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731230240576 2024-11-10T09:17:24,313 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 994da08b25124ffd8799c65e2d2ea0e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230241008 2024-11-10T09:17:24,314 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a1e2ad512746059133f9c8de74a6b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230243215 2024-11-10T09:17:24,316 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into bf3af1641acf496db8e72d1234927fa0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:24,316 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:24,316 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230244183; duration=0sec 2024-11-10T09:17:24,317 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:24,317 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:24,335 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:24,336 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/9b39cd11c8bf489284bd2a2125136095 is 50, key is test_row_0/C:col10/1731230243329/Put/seqid=0 2024-11-10T09:17:24,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742138_1314 (size=12301) 2024-11-10T09:17:24,340 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/72b0f40612ed4bf09922f5aa0cb139b4 2024-11-10T09:17:24,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742139_1315 (size=13017) 2024-11-10T09:17:24,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c8c4f06e39ae40139dcb4d3dc9908649 is 50, key is test_row_0/C:col10/1731230243388/Put/seqid=0 2024-11-10T09:17:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742140_1316 (size=12301) 2024-11-10T09:17:24,384 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c8c4f06e39ae40139dcb4d3dc9908649 2024-11-10T09:17:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/dec8169b84a145a6966449e1f06db51b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/dec8169b84a145a6966449e1f06db51b 2024-11-10T09:17:24,395 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/dec8169b84a145a6966449e1f06db51b, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T09:17:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/72b0f40612ed4bf09922f5aa0cb139b4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/72b0f40612ed4bf09922f5aa0cb139b4 2024-11-10T09:17:24,402 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/72b0f40612ed4bf09922f5aa0cb139b4, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T09:17:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/c8c4f06e39ae40139dcb4d3dc9908649 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c8c4f06e39ae40139dcb4d3dc9908649 2024-11-10T09:17:24,409 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c8c4f06e39ae40139dcb4d3dc9908649, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T09:17:24,411 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for b031bdc33b2db2f1f99a0ca67fb65c56 in 219ms, sequenceid=292, compaction requested=false 2024-11-10T09:17:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T09:17:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T09:17:24,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T09:17:24,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 833 msec 2024-11-10T09:17:24,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 841 msec 2024-11-10T09:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:24,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:17:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:24,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/1c13f5579ae54cfda0fe5dadd704a2e0 is 50, key is test_row_0/A:col10/1731230244545/Put/seqid=0 2024-11-10T09:17:24,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742141_1317 (size=14741) 2024-11-10T09:17:24,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230244591, max timestamp=1731230244616Num uniques 859; 2024-11-10T09:17:24,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230244591, max timestamp=1731230244616Num uniques 859; 2024-11-10T09:17:24,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:24,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:24,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230244591, max timestamp=1731230244616Num uniques 859; 2024-11-10T09:17:24,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:24,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230304672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:24,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230304673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:17:24,717 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:24,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T09:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:24,719 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:24,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:24,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:24,747 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/9b39cd11c8bf489284bd2a2125136095 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/9b39cd11c8bf489284bd2a2125136095 2024-11-10T09:17:24,751 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into 9b39cd11c8bf489284bd2a2125136095(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:24,751 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:24,751 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230244183; duration=0sec 2024-11-10T09:17:24,752 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:24,752 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230304787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230304787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:24,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:24,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:25,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230304997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230304997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:25,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/1c13f5579ae54cfda0fe5dadd704a2e0 2024-11-10T09:17:25,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:25,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:25,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/ad2b9a71ca114492bbb65e380ce619d2 is 50, key is test_row_0/B:col10/1731230244545/Put/seqid=0 2024-11-10T09:17:25,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742142_1318 (size=12301) 2024-11-10T09:17:25,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:25,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:25,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230305307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230305307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:25,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:25,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/ad2b9a71ca114492bbb65e380ce619d2 2024-11-10T09:17:25,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7cf9ad583efd465a9014bf10a1960ab9 is 50, key is test_row_0/C:col10/1731230244545/Put/seqid=0 2024-11-10T09:17:25,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742143_1319 (size=12301) 2024-11-10T09:17:25,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230305818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:25,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230305827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:25,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:25,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:25,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7cf9ad583efd465a9014bf10a1960ab9 2024-11-10T09:17:25,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/1c13f5579ae54cfda0fe5dadd704a2e0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/1c13f5579ae54cfda0fe5dadd704a2e0 2024-11-10T09:17:25,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/1c13f5579ae54cfda0fe5dadd704a2e0, entries=200, sequenceid=304, filesize=14.4 K 2024-11-10T09:17:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/ad2b9a71ca114492bbb65e380ce619d2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ad2b9a71ca114492bbb65e380ce619d2 2024-11-10T09:17:25,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ad2b9a71ca114492bbb65e380ce619d2, entries=150, sequenceid=304, filesize=12.0 K 2024-11-10T09:17:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7cf9ad583efd465a9014bf10a1960ab9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7cf9ad583efd465a9014bf10a1960ab9 2024-11-10T09:17:25,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7cf9ad583efd465a9014bf10a1960ab9, entries=150, sequenceid=304, filesize=12.0 K 2024-11-10T09:17:25,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1362ms, sequenceid=304, compaction requested=true 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:25,913 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:25,913 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:25,914 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:25,914 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:25,914 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:25,914 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:25,914 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,914 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,914 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/bf3af1641acf496db8e72d1234927fa0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/72b0f40612ed4bf09922f5aa0cb139b4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ad2b9a71ca114492bbb65e380ce619d2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.7 K 2024-11-10T09:17:25,914 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/11b6aaf8b9794300bd0392f7767b67b5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/dec8169b84a145a6966449e1f06db51b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/1c13f5579ae54cfda0fe5dadd704a2e0] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=39.1 K 2024-11-10T09:17:25,915 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3af1641acf496db8e72d1234927fa0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230243215 2024-11-10T09:17:25,915 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b6aaf8b9794300bd0392f7767b67b5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230243215 2024-11-10T09:17:25,915 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b0f40612ed4bf09922f5aa0cb139b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230243383 2024-11-10T09:17:25,915 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting dec8169b84a145a6966449e1f06db51b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230243383 2024-11-10T09:17:25,915 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2b9a71ca114492bbb65e380ce619d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731230244544 2024-11-10T09:17:25,915 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c13f5579ae54cfda0fe5dadd704a2e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731230244535 2024-11-10T09:17:25,924 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:25,924 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:25,924 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/f44cf0ce884c4dc3b77599dace729545 is 50, key is test_row_0/B:col10/1731230244545/Put/seqid=0 2024-11-10T09:17:25,924 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/3c6d5eaf47d4419fa42a89a8c1b78013 is 50, key is test_row_0/A:col10/1731230244545/Put/seqid=0 2024-11-10T09:17:25,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742144_1320 (size=13119) 2024-11-10T09:17:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742145_1321 (size=13119) 2024-11-10T09:17:25,945 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/3c6d5eaf47d4419fa42a89a8c1b78013 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/3c6d5eaf47d4419fa42a89a8c1b78013 2024-11-10T09:17:25,947 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/f44cf0ce884c4dc3b77599dace729545 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f44cf0ce884c4dc3b77599dace729545 2024-11-10T09:17:25,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:17:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,953 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T09:17:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:25,955 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into f44cf0ce884c4dc3b77599dace729545(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:25,955 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230245913; duration=0sec 2024-11-10T09:17:25,955 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into 3c6d5eaf47d4419fa42a89a8c1b78013(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:25,955 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230245913; duration=0sec 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:25,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:25,957 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:25,958 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:25,958 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:25,959 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/9b39cd11c8bf489284bd2a2125136095, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c8c4f06e39ae40139dcb4d3dc9908649, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7cf9ad583efd465a9014bf10a1960ab9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.7 K 2024-11-10T09:17:25,960 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b39cd11c8bf489284bd2a2125136095, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230243215 2024-11-10T09:17:25,961 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c4f06e39ae40139dcb4d3dc9908649, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230243383 2024-11-10T09:17:25,961 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cf9ad583efd465a9014bf10a1960ab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731230244544 2024-11-10T09:17:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/20bcd8b8e1ec4016b4150996bf5f9ac5 is 50, key is test_row_0/A:col10/1731230244672/Put/seqid=0 2024-11-10T09:17:25,980 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:25,981 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7d2dc73bf6da467fb62359e16de97c02 is 50, key is test_row_0/C:col10/1731230244545/Put/seqid=0 2024-11-10T09:17:26,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742146_1322 (size=12301) 2024-11-10T09:17:26,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742147_1323 (size=13119) 2024-11-10T09:17:26,050 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/7d2dc73bf6da467fb62359e16de97c02 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7d2dc73bf6da467fb62359e16de97c02 2024-11-10T09:17:26,057 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into 7d2dc73bf6da467fb62359e16de97c02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:26,057 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:26,057 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230245913; duration=0sec 2024-11-10T09:17:26,058 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:26,058 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:26,422 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/20bcd8b8e1ec4016b4150996bf5f9ac5 2024-11-10T09:17:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/5a29d38ef79c44fe835e2709c3864085 is 50, key is test_row_0/B:col10/1731230244672/Put/seqid=0 2024-11-10T09:17:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742148_1324 (size=12301) 2024-11-10T09:17:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:26,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/5a29d38ef79c44fe835e2709c3864085 2024-11-10T09:17:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4161c3f069c3480a897178b755240040 is 50, key is test_row_0/C:col10/1731230244672/Put/seqid=0 2024-11-10T09:17:26,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230306860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742149_1325 (size=12301) 2024-11-10T09:17:26,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230306861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230306977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230306977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230307187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230307187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,266 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4161c3f069c3480a897178b755240040 2024-11-10T09:17:27,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/20bcd8b8e1ec4016b4150996bf5f9ac5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/20bcd8b8e1ec4016b4150996bf5f9ac5 2024-11-10T09:17:27,289 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/20bcd8b8e1ec4016b4150996bf5f9ac5, entries=150, sequenceid=343, filesize=12.0 K 2024-11-10T09:17:27,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/5a29d38ef79c44fe835e2709c3864085 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/5a29d38ef79c44fe835e2709c3864085 2024-11-10T09:17:27,295 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/5a29d38ef79c44fe835e2709c3864085, entries=150, sequenceid=343, filesize=12.0 K 2024-11-10T09:17:27,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4161c3f069c3480a897178b755240040 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4161c3f069c3480a897178b755240040 2024-11-10T09:17:27,302 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4161c3f069c3480a897178b755240040, entries=150, sequenceid=343, filesize=12.0 K 2024-11-10T09:17:27,302 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1350ms, sequenceid=343, compaction requested=false 2024-11-10T09:17:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T09:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T09:17:27,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T09:17:27,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5840 sec 2024-11-10T09:17:27,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.5880 sec 2024-11-10T09:17:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:27,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:17:27,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:27,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/ea45e24b418a47e3870ca011a2ec0ea0 is 50, key is test_row_0/A:col10/1731230246852/Put/seqid=0 2024-11-10T09:17:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742150_1326 (size=14741) 2024-11-10T09:17:27,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230307558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230307563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230307667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230307677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230307877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230307887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:27,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/ea45e24b418a47e3870ca011a2ec0ea0 2024-11-10T09:17:27,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/8bc4d25e8b774484b4d915a257ac62f9 is 50, key is test_row_0/B:col10/1731230246852/Put/seqid=0 2024-11-10T09:17:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742151_1327 (size=12301) 2024-11-10T09:17:27,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/8bc4d25e8b774484b4d915a257ac62f9 2024-11-10T09:17:27,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8e054bef1f124c5da34182c29cb9af0b is 50, key is test_row_0/C:col10/1731230246852/Put/seqid=0 2024-11-10T09:17:27,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742152_1328 (size=12301) 2024-11-10T09:17:27,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8e054bef1f124c5da34182c29cb9af0b 2024-11-10T09:17:27,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/ea45e24b418a47e3870ca011a2ec0ea0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ea45e24b418a47e3870ca011a2ec0ea0 2024-11-10T09:17:27,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ea45e24b418a47e3870ca011a2ec0ea0, entries=200, sequenceid=357, filesize=14.4 K 2024-11-10T09:17:27,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/8bc4d25e8b774484b4d915a257ac62f9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8bc4d25e8b774484b4d915a257ac62f9 2024-11-10T09:17:27,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8bc4d25e8b774484b4d915a257ac62f9, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T09:17:27,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/8e054bef1f124c5da34182c29cb9af0b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e054bef1f124c5da34182c29cb9af0b 2024-11-10T09:17:27,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e054bef1f124c5da34182c29cb9af0b, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T09:17:27,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b031bdc33b2db2f1f99a0ca67fb65c56 in 497ms, sequenceid=357, compaction requested=true 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:27,998 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:27,998 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:27,999 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:27,999 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:27,999 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:27,999 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:27,999 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:27,999 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:27,999 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f44cf0ce884c4dc3b77599dace729545, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/5a29d38ef79c44fe835e2709c3864085, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8bc4d25e8b774484b4d915a257ac62f9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.8 K 2024-11-10T09:17:27,999 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/3c6d5eaf47d4419fa42a89a8c1b78013, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/20bcd8b8e1ec4016b4150996bf5f9ac5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ea45e24b418a47e3870ca011a2ec0ea0] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=39.2 K 2024-11-10T09:17:28,000 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f44cf0ce884c4dc3b77599dace729545, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731230244544 2024-11-10T09:17:28,000 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6d5eaf47d4419fa42a89a8c1b78013, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731230244544 2024-11-10T09:17:28,000 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20bcd8b8e1ec4016b4150996bf5f9ac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731230244664 2024-11-10T09:17:28,000 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a29d38ef79c44fe835e2709c3864085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731230244664 2024-11-10T09:17:28,000 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc4d25e8b774484b4d915a257ac62f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731230246852 2024-11-10T09:17:28,000 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea45e24b418a47e3870ca011a2ec0ea0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731230246852 2024-11-10T09:17:28,009 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:28,009 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/8a806e6b9d484fd091c2ade10e77948f is 50, key is test_row_0/B:col10/1731230246852/Put/seqid=0 2024-11-10T09:17:28,009 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:28,010 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/f1dae107f9a142d4b7fd9422db817714 is 50, key is test_row_0/A:col10/1731230246852/Put/seqid=0 2024-11-10T09:17:28,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742153_1329 (size=13221) 2024-11-10T09:17:28,032 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/8a806e6b9d484fd091c2ade10e77948f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8a806e6b9d484fd091c2ade10e77948f 2024-11-10T09:17:28,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742154_1330 (size=13221) 2024-11-10T09:17:28,039 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into 8a806e6b9d484fd091c2ade10e77948f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:28,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:28,039 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230247998; duration=0sec 2024-11-10T09:17:28,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:28,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:28,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:28,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:28,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:28,041 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:28,041 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7d2dc73bf6da467fb62359e16de97c02, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4161c3f069c3480a897178b755240040, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e054bef1f124c5da34182c29cb9af0b] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.8 K 2024-11-10T09:17:28,042 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2dc73bf6da467fb62359e16de97c02, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731230244544 2024-11-10T09:17:28,042 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4161c3f069c3480a897178b755240040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731230244664 2024-11-10T09:17:28,043 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e054bef1f124c5da34182c29cb9af0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731230246852 2024-11-10T09:17:28,044 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/f1dae107f9a142d4b7fd9422db817714 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/f1dae107f9a142d4b7fd9422db817714 2024-11-10T09:17:28,051 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into f1dae107f9a142d4b7fd9422db817714(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:28,051 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:28,051 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230247998; duration=0sec 2024-11-10T09:17:28,051 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:28,051 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:28,052 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:28,053 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/de4fc2e2c0e24453af277cb5c9444c71 is 50, key is test_row_0/C:col10/1731230246852/Put/seqid=0 2024-11-10T09:17:28,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742155_1331 (size=13221) 2024-11-10T09:17:28,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230247533, max timestamp=1731230248187Num uniques 859; 2024-11-10T09:17:28,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:28,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230247533, max timestamp=1731230248187Num uniques 859; 2024-11-10T09:17:28,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:28,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-10T09:17:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:28,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/a2b9ed3179e74e21b6585e33af2bb63f is 50, key is test_row_0/A:col10/1731230247533/Put/seqid=0 2024-11-10T09:17:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742156_1332 (size=14741) 2024-11-10T09:17:28,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230248245, max timestamp=1731230248260Num uniques 859; 2024-11-10T09:17:28,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:28,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230248245, max timestamp=1731230248260Num uniques 859; 2024-11-10T09:17:28,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230248245, max timestamp=1731230248260Num uniques 859; 2024-11-10T09:17:28,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:17:28,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:17:28,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230308330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230308330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230308447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230308447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,460 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/de4fc2e2c0e24453af277cb5c9444c71 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/de4fc2e2c0e24453af277cb5c9444c71 2024-11-10T09:17:28,466 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into de4fc2e2c0e24453af277cb5c9444c71(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:28,466 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:28,466 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230247998; duration=0sec 2024-11-10T09:17:28,467 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:28,467 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:28,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/a2b9ed3179e74e21b6585e33af2bb63f 2024-11-10T09:17:28,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/f173fca5ad144eb2a9d2da66952bab6a is 50, key is test_row_0/B:col10/1731230247533/Put/seqid=0 2024-11-10T09:17:28,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742157_1333 (size=12301) 2024-11-10T09:17:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230308657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230308657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:17:28,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:28,897 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T09:17:28,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:28,901 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:28,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230308967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230308968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:29,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/f173fca5ad144eb2a9d2da66952bab6a 2024-11-10T09:17:29,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/58de35e00d624edbaf4bef2eb86c554c is 50, key is test_row_0/C:col10/1731230247533/Put/seqid=0 2024-11-10T09:17:29,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742158_1334 (size=12301) 2024-11-10T09:17:29,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:29,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/58de35e00d624edbaf4bef2eb86c554c 2024-11-10T09:17:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/a2b9ed3179e74e21b6585e33af2bb63f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/a2b9ed3179e74e21b6585e33af2bb63f 2024-11-10T09:17:29,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/a2b9ed3179e74e21b6585e33af2bb63f, entries=200, sequenceid=380, filesize=14.4 K 2024-11-10T09:17:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/f173fca5ad144eb2a9d2da66952bab6a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f173fca5ad144eb2a9d2da66952bab6a 2024-11-10T09:17:29,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f173fca5ad144eb2a9d2da66952bab6a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-10T09:17:29,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/58de35e00d624edbaf4bef2eb86c554c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/58de35e00d624edbaf4bef2eb86c554c 2024-11-10T09:17:29,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/58de35e00d624edbaf4bef2eb86c554c, entries=150, sequenceid=380, filesize=12.0 K 2024-11-10T09:17:29,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=126.68 KB/129720 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1253ms, sequenceid=380, compaction requested=false 2024-11-10T09:17:29,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:29,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-11-10T09:17:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:29,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/92419f2ace9b442c88c5999edef2d979 is 50, key is test_row_0/A:col10/1731230248304/Put/seqid=0 2024-11-10T09:17:29,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742159_1335 (size=12301) 2024-11-10T09:17:29,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230309511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230309513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:29,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230309627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230309628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52522 deadline: 1731230309637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,648 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52516 deadline: 1731230309657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,663 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18309 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52442 deadline: 1731230309667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,670 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in b031bdc33b2db2f1f99a0ca67fb65c56 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18318 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230309838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230309839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:29,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/92419f2ace9b442c88c5999edef2d979 2024-11-10T09:17:29,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/2eae9bb94f7c429bb804f41b0c25bcf4 is 50, key is test_row_0/B:col10/1731230248304/Put/seqid=0 2024-11-10T09:17:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742160_1336 (size=12301) 2024-11-10T09:17:29,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/2eae9bb94f7c429bb804f41b0c25bcf4 2024-11-10T09:17:29,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/163243db2d014d058a7090e7050d8b7d is 50, key is test_row_0/C:col10/1731230248304/Put/seqid=0 2024-11-10T09:17:29,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:29,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742161_1337 (size=12301) 2024-11-10T09:17:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:30,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:30,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:30,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52450 deadline: 1731230310147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:30,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52440 deadline: 1731230310157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:17:30,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/163243db2d014d058a7090e7050d8b7d 2024-11-10T09:17:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/92419f2ace9b442c88c5999edef2d979 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/92419f2ace9b442c88c5999edef2d979 2024-11-10T09:17:30,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/92419f2ace9b442c88c5999edef2d979, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T09:17:30,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/2eae9bb94f7c429bb804f41b0c25bcf4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2eae9bb94f7c429bb804f41b0c25bcf4 2024-11-10T09:17:30,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2eae9bb94f7c429bb804f41b0c25bcf4, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T09:17:30,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/163243db2d014d058a7090e7050d8b7d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/163243db2d014d058a7090e7050d8b7d 2024-11-10T09:17:30,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:30,462 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-10T09:17:30,462 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,462 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-10T09:17:30,462 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,462 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,463 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,463 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,463 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,464 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-10T09:17:30,464 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,464 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,464 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,467 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-10T09:17:30,467 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-10T09:17:30,467 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,467 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,467 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,467 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,467 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,467 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/163243db2d014d058a7090e7050d8b7d, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T09:17:30,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.39 KB/136590, heapSize ~383.11 KB/392304, currentSize=46.96 KB/48090 for b031bdc33b2db2f1f99a0ca67fb65c56 in 990ms, sequenceid=418, compaction requested=true 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:30,470 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:30,470 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b031bdc33b2db2f1f99a0ca67fb65c56:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:30,471 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:30,471 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:30,471 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/B is initiating minor compaction (all files) 2024-11-10T09:17:30,471 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/A is initiating minor compaction (all files) 2024-11-10T09:17:30,471 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/B in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,471 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/A in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,471 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8a806e6b9d484fd091c2ade10e77948f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f173fca5ad144eb2a9d2da66952bab6a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2eae9bb94f7c429bb804f41b0c25bcf4] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.9 K 2024-11-10T09:17:30,471 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/f1dae107f9a142d4b7fd9422db817714, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/a2b9ed3179e74e21b6585e33af2bb63f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/92419f2ace9b442c88c5999edef2d979] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=39.3 K 2024-11-10T09:17:30,471 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1dae107f9a142d4b7fd9422db817714, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731230246852 2024-11-10T09:17:30,471 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a806e6b9d484fd091c2ade10e77948f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731230246852 2024-11-10T09:17:30,472 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f173fca5ad144eb2a9d2da66952bab6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731230247533 2024-11-10T09:17:30,472 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2b9ed3179e74e21b6585e33af2bb63f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731230247533 2024-11-10T09:17:30,472 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eae9bb94f7c429bb804f41b0c25bcf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731230248304 2024-11-10T09:17:30,472 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92419f2ace9b442c88c5999edef2d979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731230248304 2024-11-10T09:17:30,479 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:30,479 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#B#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:30,480 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/0ac88ad9bef14bda81eda4eb23fc71f3 is 50, key is test_row_0/A:col10/1731230248304/Put/seqid=0 2024-11-10T09:17:30,480 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/3c0cbe1426b748969197067acbd2a2bd is 50, key is test_row_0/B:col10/1731230248304/Put/seqid=0 2024-11-10T09:17:30,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742162_1338 (size=13323) 2024-11-10T09:17:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742163_1339 (size=13323) 2024-11-10T09:17:30,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:17:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,614 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:17:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:30,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/efb2e35120544b42846024311a45ff0c is 50, key is test_row_1/A:col10/1731230249501/Put/seqid=0 2024-11-10T09:17:30,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742164_1340 (size=9857) 2024-11-10T09:17:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:30,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. as already flushing 2024-11-10T09:17:30,659 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-10T09:17:30,659 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,659 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,659 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,682 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-10T09:17:30,682 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:30,682 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,682 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:30,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:30,888 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/0ac88ad9bef14bda81eda4eb23fc71f3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/0ac88ad9bef14bda81eda4eb23fc71f3 2024-11-10T09:17:30,888 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/3c0cbe1426b748969197067acbd2a2bd as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/3c0cbe1426b748969197067acbd2a2bd 2024-11-10T09:17:30,892 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/A of b031bdc33b2db2f1f99a0ca67fb65c56 into 0ac88ad9bef14bda81eda4eb23fc71f3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:30,892 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/B of b031bdc33b2db2f1f99a0ca67fb65c56 into 3c0cbe1426b748969197067acbd2a2bd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:30,893 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/A, priority=13, startTime=1731230250470; duration=0sec 2024-11-10T09:17:30,893 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/B, priority=13, startTime=1731230250470; duration=0sec 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:A 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:B 2024-11-10T09:17:30,893 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:30,894 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:30,894 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): b031bdc33b2db2f1f99a0ca67fb65c56/C is initiating minor compaction (all files) 2024-11-10T09:17:30,894 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b031bdc33b2db2f1f99a0ca67fb65c56/C in TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:30,894 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/de4fc2e2c0e24453af277cb5c9444c71, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/58de35e00d624edbaf4bef2eb86c554c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/163243db2d014d058a7090e7050d8b7d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp, totalSize=36.9 K 2024-11-10T09:17:30,894 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4fc2e2c0e24453af277cb5c9444c71, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731230246852 2024-11-10T09:17:30,895 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58de35e00d624edbaf4bef2eb86c554c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731230247533 2024-11-10T09:17:30,895 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163243db2d014d058a7090e7050d8b7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731230248304 2024-11-10T09:17:30,904 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b031bdc33b2db2f1f99a0ca67fb65c56#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:30,905 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/f2f8e8b096c149f3a82f1a72390ce61f is 50, key is test_row_0/C:col10/1731230248304/Put/seqid=0 2024-11-10T09:17:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742165_1341 (size=13323) 2024-11-10T09:17:31,024 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/efb2e35120544b42846024311a45ff0c 2024-11-10T09:17:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/26b06ae3388f4e8980fbd71855f1249f is 50, key is test_row_1/B:col10/1731230249501/Put/seqid=0 2024-11-10T09:17:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:31,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742166_1342 (size=9857) 2024-11-10T09:17:31,315 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/f2f8e8b096c149f3a82f1a72390ce61f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/f2f8e8b096c149f3a82f1a72390ce61f 2024-11-10T09:17:31,321 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b031bdc33b2db2f1f99a0ca67fb65c56/C of b031bdc33b2db2f1f99a0ca67fb65c56 into f2f8e8b096c149f3a82f1a72390ce61f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:31,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:31,321 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., storeName=b031bdc33b2db2f1f99a0ca67fb65c56/C, priority=13, startTime=1731230250470; duration=0sec 2024-11-10T09:17:31,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:31,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b031bdc33b2db2f1f99a0ca67fb65c56:C 2024-11-10T09:17:31,609 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/26b06ae3388f4e8980fbd71855f1249f 2024-11-10T09:17:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/64f3d7239cbc41a9ac3fc507819aa3f2 is 50, key is test_row_1/C:col10/1731230249501/Put/seqid=0 2024-11-10T09:17:31,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742167_1343 (size=9857) 2024-11-10T09:17:32,020 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/64f3d7239cbc41a9ac3fc507819aa3f2 2024-11-10T09:17:32,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/efb2e35120544b42846024311a45ff0c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/efb2e35120544b42846024311a45ff0c 2024-11-10T09:17:32,031 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/efb2e35120544b42846024311a45ff0c, entries=100, sequenceid=428, filesize=9.6 K 2024-11-10T09:17:32,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/26b06ae3388f4e8980fbd71855f1249f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/26b06ae3388f4e8980fbd71855f1249f 2024-11-10T09:17:32,045 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/26b06ae3388f4e8980fbd71855f1249f, entries=100, sequenceid=428, filesize=9.6 K 2024-11-10T09:17:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/64f3d7239cbc41a9ac3fc507819aa3f2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/64f3d7239cbc41a9ac3fc507819aa3f2 2024-11-10T09:17:32,052 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/64f3d7239cbc41a9ac3fc507819aa3f2, entries=100, sequenceid=428, filesize=9.6 K 2024-11-10T09:17:32,053 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1439ms, sequenceid=428, compaction requested=false 2024-11-10T09:17:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b031bdc33b2db2f1f99a0ca67fb65c56: 2024-11-10T09:17:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T09:17:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T09:17:32,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T09:17:32,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1540 sec 2024-11-10T09:17:32,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 3.1610 sec 2024-11-10T09:17:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:17:33,068 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:36,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:17:39,709 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-10T09:17:39,709 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:39,709 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,709 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:39,719 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-10T09:17:39,719 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:39,719 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,719 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:39,730 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-10T09:17:39,730 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:17:39,730 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,730 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 160 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2637 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2735 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2579 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2642 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2763 2024-11-10T09:17:39,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:17:39,731 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:17:39,731 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:17:39,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:39,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:17:39,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:17:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:17:39,737 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230259737"}]},"ts":"1731230259737"} 2024-11-10T09:17:39,739 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:17:39,739 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:17:39,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:17:39,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, UNASSIGN}] 2024-11-10T09:17:39,743 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, UNASSIGN 2024-11-10T09:17:39,744 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b031bdc33b2db2f1f99a0ca67fb65c56, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:39,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, UNASSIGN because future has completed 2024-11-10T09:17:39,746 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:17:39,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:17:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:17:39,898 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:39,898 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing b031bdc33b2db2f1f99a0ca67fb65c56, disabling compactions & flushes 2024-11-10T09:17:39,899 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. after waiting 0 ms 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:39,899 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing b031bdc33b2db2f1f99a0ca67fb65c56 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=A 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=B 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b031bdc33b2db2f1f99a0ca67fb65c56, store=C 2024-11-10T09:17:39,899 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:39,903 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/239b5c9daa7d4bb297dfb3fa32ee6edf is 50, key is test_row_0/A:col10/1731230259728/Put/seqid=0 2024-11-10T09:17:39,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742168_1344 (size=9857) 2024-11-10T09:17:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:17:40,307 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/239b5c9daa7d4bb297dfb3fa32ee6edf 2024-11-10T09:17:40,315 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b2586505f8264348b673748b52e35ee3 is 50, key is test_row_0/B:col10/1731230259728/Put/seqid=0 2024-11-10T09:17:40,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742169_1345 (size=9857) 2024-11-10T09:17:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:17:40,720 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b2586505f8264348b673748b52e35ee3 2024-11-10T09:17:40,730 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4e6a83c579bb46e0ae527a26f3d19ff3 is 50, key is test_row_0/C:col10/1731230259728/Put/seqid=0 2024-11-10T09:17:40,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742170_1346 (size=9857) 2024-11-10T09:17:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:17:41,137 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4e6a83c579bb46e0ae527a26f3d19ff3 2024-11-10T09:17:41,145 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/A/239b5c9daa7d4bb297dfb3fa32ee6edf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/239b5c9daa7d4bb297dfb3fa32ee6edf 2024-11-10T09:17:41,148 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/239b5c9daa7d4bb297dfb3fa32ee6edf, entries=100, sequenceid=439, filesize=9.6 K 2024-11-10T09:17:41,149 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/B/b2586505f8264348b673748b52e35ee3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b2586505f8264348b673748b52e35ee3 2024-11-10T09:17:41,151 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b2586505f8264348b673748b52e35ee3, entries=100, sequenceid=439, filesize=9.6 K 2024-11-10T09:17:41,152 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/.tmp/C/4e6a83c579bb46e0ae527a26f3d19ff3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4e6a83c579bb46e0ae527a26f3d19ff3 2024-11-10T09:17:41,155 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4e6a83c579bb46e0ae527a26f3d19ff3, entries=100, sequenceid=439, filesize=9.6 K 2024-11-10T09:17:41,155 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1256ms, sequenceid=439, compaction requested=true 2024-11-10T09:17:41,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/436928103a154e80a3274aa51a7f606b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8d169065e7de46cca79db6f1af3b8c4a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/9acec32ccd5442bfabe9bf5c2415bc9c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/98de9732995543c48182531010212b21, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/d0e9d612008e41d695d57302cc312c97, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8fa38dfbc0034856bbaa5759c66bea00, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ee6c472de0394a88b13036d470c1119d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6b60e22375a9473fa1a6ae3fc4e619be, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/bf3bd64f718c49fc86d7ae2253133287, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/7da9ee7e918d47e39a5bf1301e4bb3be, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/062dabcff88b45fd88a70541a26f539d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/db29550a111f454b8ebb5ec60c0e453e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/fbe2c920acad45c2a86dfc92dd50d9b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/50dfe81850bb4b6ca57883b7b9a7017f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/df88a725a6cf49caae8b2b517a8fb449, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/93fdfb35bf824c15b601c120dcf72d00, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8517829f4ac74e908b75baa52ecf680b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/b4b601b96fd144fc837cc7239064c135, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/c22031829420425baa10941cb41fe2e3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/035c493c74134913a2c0e108e8e24484, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6ccd7bbad4024b079bcdc9335191eab9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/11b6aaf8b9794300bd0392f7767b67b5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/dec8169b84a145a6966449e1f06db51b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/1c13f5579ae54cfda0fe5dadd704a2e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/3c6d5eaf47d4419fa42a89a8c1b78013, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/20bcd8b8e1ec4016b4150996bf5f9ac5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ea45e24b418a47e3870ca011a2ec0ea0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/f1dae107f9a142d4b7fd9422db817714, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/a2b9ed3179e74e21b6585e33af2bb63f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/92419f2ace9b442c88c5999edef2d979] to archive 2024-11-10T09:17:41,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:17:41,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/436928103a154e80a3274aa51a7f606b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/436928103a154e80a3274aa51a7f606b 2024-11-10T09:17:41,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8d169065e7de46cca79db6f1af3b8c4a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8d169065e7de46cca79db6f1af3b8c4a 2024-11-10T09:17:41,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/9acec32ccd5442bfabe9bf5c2415bc9c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/9acec32ccd5442bfabe9bf5c2415bc9c 2024-11-10T09:17:41,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/98de9732995543c48182531010212b21 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/98de9732995543c48182531010212b21 2024-11-10T09:17:41,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/d0e9d612008e41d695d57302cc312c97 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/d0e9d612008e41d695d57302cc312c97 2024-11-10T09:17:41,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8fa38dfbc0034856bbaa5759c66bea00 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8fa38dfbc0034856bbaa5759c66bea00 2024-11-10T09:17:41,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ee6c472de0394a88b13036d470c1119d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ee6c472de0394a88b13036d470c1119d 2024-11-10T09:17:41,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6b60e22375a9473fa1a6ae3fc4e619be to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6b60e22375a9473fa1a6ae3fc4e619be 2024-11-10T09:17:41,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/bf3bd64f718c49fc86d7ae2253133287 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/bf3bd64f718c49fc86d7ae2253133287 2024-11-10T09:17:41,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/7da9ee7e918d47e39a5bf1301e4bb3be to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/7da9ee7e918d47e39a5bf1301e4bb3be 2024-11-10T09:17:41,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/062dabcff88b45fd88a70541a26f539d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/062dabcff88b45fd88a70541a26f539d 2024-11-10T09:17:41,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/db29550a111f454b8ebb5ec60c0e453e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/db29550a111f454b8ebb5ec60c0e453e 2024-11-10T09:17:41,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/fbe2c920acad45c2a86dfc92dd50d9b1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/fbe2c920acad45c2a86dfc92dd50d9b1 2024-11-10T09:17:41,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/50dfe81850bb4b6ca57883b7b9a7017f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/50dfe81850bb4b6ca57883b7b9a7017f 2024-11-10T09:17:41,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/df88a725a6cf49caae8b2b517a8fb449 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/df88a725a6cf49caae8b2b517a8fb449 2024-11-10T09:17:41,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/93fdfb35bf824c15b601c120dcf72d00 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/93fdfb35bf824c15b601c120dcf72d00 2024-11-10T09:17:41,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8517829f4ac74e908b75baa52ecf680b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/8517829f4ac74e908b75baa52ecf680b 2024-11-10T09:17:41,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/b4b601b96fd144fc837cc7239064c135 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/b4b601b96fd144fc837cc7239064c135 2024-11-10T09:17:41,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/c22031829420425baa10941cb41fe2e3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/c22031829420425baa10941cb41fe2e3 2024-11-10T09:17:41,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/035c493c74134913a2c0e108e8e24484 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/035c493c74134913a2c0e108e8e24484 2024-11-10T09:17:41,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6ccd7bbad4024b079bcdc9335191eab9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/6ccd7bbad4024b079bcdc9335191eab9 2024-11-10T09:17:41,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/11b6aaf8b9794300bd0392f7767b67b5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/11b6aaf8b9794300bd0392f7767b67b5 2024-11-10T09:17:41,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/dec8169b84a145a6966449e1f06db51b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/dec8169b84a145a6966449e1f06db51b 2024-11-10T09:17:41,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/1c13f5579ae54cfda0fe5dadd704a2e0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/1c13f5579ae54cfda0fe5dadd704a2e0 2024-11-10T09:17:41,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/3c6d5eaf47d4419fa42a89a8c1b78013 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/3c6d5eaf47d4419fa42a89a8c1b78013 2024-11-10T09:17:41,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/20bcd8b8e1ec4016b4150996bf5f9ac5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/20bcd8b8e1ec4016b4150996bf5f9ac5 2024-11-10T09:17:41,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ea45e24b418a47e3870ca011a2ec0ea0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/ea45e24b418a47e3870ca011a2ec0ea0 2024-11-10T09:17:41,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/f1dae107f9a142d4b7fd9422db817714 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/f1dae107f9a142d4b7fd9422db817714 2024-11-10T09:17:41,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/a2b9ed3179e74e21b6585e33af2bb63f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/a2b9ed3179e74e21b6585e33af2bb63f 2024-11-10T09:17:41,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/92419f2ace9b442c88c5999edef2d979 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/92419f2ace9b442c88c5999edef2d979 2024-11-10T09:17:41,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/1c62ea4e3e9a487489fd52e5f0207d50, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/427dea98fecc4672bdaded703a76f6ad, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/d87c38488db24936978ff5647a8259c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/95e9c377aa954bc586f1c5b0b1cb415a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/0ee55efd98214997b84ef388ef676cc1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4a2d55e5d79d4f679cf7a58a8c0125e4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2a5dd3f67a55468793e0204829ece1f6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/31cee670beaf4f26a785f6a26e66a7aa, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e55dbeb9cff74337b0b1aab518670a2d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4cd540fb564d4b91bf3a3bdc885fb384, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/fa57f9242ec94aefa32a2882705c19f8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/7f44d043f9c443fb9eb4fdf678295830, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/65d5dd4f19dc40d69199d0df2a868467, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ef9192d2c6f6498b97d5cee619a07260, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/c232a0fbbd8e49108df4b2604b242b8c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b145ef70452d4d5caafdba6789ddd3a2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4b4b520f918d4918aa214d46bc5fd885, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/beadd40876e14850a6bfc58f76711aee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e5a3fa590f6641a38bcc64106a45462d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b1ce6d63af59437da766fd39b8fd7043, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/bf3af1641acf496db8e72d1234927fa0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/9c9a4e47ff444069b89bb896873beed6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/72b0f40612ed4bf09922f5aa0cb139b4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f44cf0ce884c4dc3b77599dace729545, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ad2b9a71ca114492bbb65e380ce619d2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/5a29d38ef79c44fe835e2709c3864085, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8a806e6b9d484fd091c2ade10e77948f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8bc4d25e8b774484b4d915a257ac62f9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f173fca5ad144eb2a9d2da66952bab6a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2eae9bb94f7c429bb804f41b0c25bcf4] to archive 2024-11-10T09:17:41,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:17:41,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/1c62ea4e3e9a487489fd52e5f0207d50 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/1c62ea4e3e9a487489fd52e5f0207d50 2024-11-10T09:17:41,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/427dea98fecc4672bdaded703a76f6ad to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/427dea98fecc4672bdaded703a76f6ad 2024-11-10T09:17:41,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/d87c38488db24936978ff5647a8259c0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/d87c38488db24936978ff5647a8259c0 2024-11-10T09:17:41,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/95e9c377aa954bc586f1c5b0b1cb415a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/95e9c377aa954bc586f1c5b0b1cb415a 2024-11-10T09:17:41,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/0ee55efd98214997b84ef388ef676cc1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/0ee55efd98214997b84ef388ef676cc1 2024-11-10T09:17:41,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4a2d55e5d79d4f679cf7a58a8c0125e4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4a2d55e5d79d4f679cf7a58a8c0125e4 2024-11-10T09:17:41,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2a5dd3f67a55468793e0204829ece1f6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2a5dd3f67a55468793e0204829ece1f6 2024-11-10T09:17:41,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/31cee670beaf4f26a785f6a26e66a7aa to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/31cee670beaf4f26a785f6a26e66a7aa 2024-11-10T09:17:41,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e55dbeb9cff74337b0b1aab518670a2d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e55dbeb9cff74337b0b1aab518670a2d 2024-11-10T09:17:41,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4cd540fb564d4b91bf3a3bdc885fb384 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4cd540fb564d4b91bf3a3bdc885fb384 2024-11-10T09:17:41,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/fa57f9242ec94aefa32a2882705c19f8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/fa57f9242ec94aefa32a2882705c19f8 2024-11-10T09:17:41,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/7f44d043f9c443fb9eb4fdf678295830 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/7f44d043f9c443fb9eb4fdf678295830 2024-11-10T09:17:41,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/65d5dd4f19dc40d69199d0df2a868467 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/65d5dd4f19dc40d69199d0df2a868467 2024-11-10T09:17:41,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ef9192d2c6f6498b97d5cee619a07260 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ef9192d2c6f6498b97d5cee619a07260 2024-11-10T09:17:41,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/c232a0fbbd8e49108df4b2604b242b8c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/c232a0fbbd8e49108df4b2604b242b8c 2024-11-10T09:17:41,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b145ef70452d4d5caafdba6789ddd3a2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b145ef70452d4d5caafdba6789ddd3a2 2024-11-10T09:17:41,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4b4b520f918d4918aa214d46bc5fd885 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/4b4b520f918d4918aa214d46bc5fd885 2024-11-10T09:17:41,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/beadd40876e14850a6bfc58f76711aee to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/beadd40876e14850a6bfc58f76711aee 2024-11-10T09:17:41,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e5a3fa590f6641a38bcc64106a45462d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/e5a3fa590f6641a38bcc64106a45462d 2024-11-10T09:17:41,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b1ce6d63af59437da766fd39b8fd7043 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b1ce6d63af59437da766fd39b8fd7043 2024-11-10T09:17:41,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/bf3af1641acf496db8e72d1234927fa0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/bf3af1641acf496db8e72d1234927fa0 2024-11-10T09:17:41,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/9c9a4e47ff444069b89bb896873beed6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/9c9a4e47ff444069b89bb896873beed6 2024-11-10T09:17:41,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/72b0f40612ed4bf09922f5aa0cb139b4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/72b0f40612ed4bf09922f5aa0cb139b4 2024-11-10T09:17:41,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f44cf0ce884c4dc3b77599dace729545 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f44cf0ce884c4dc3b77599dace729545 2024-11-10T09:17:41,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ad2b9a71ca114492bbb65e380ce619d2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/ad2b9a71ca114492bbb65e380ce619d2 2024-11-10T09:17:41,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/5a29d38ef79c44fe835e2709c3864085 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/5a29d38ef79c44fe835e2709c3864085 2024-11-10T09:17:41,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8a806e6b9d484fd091c2ade10e77948f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8a806e6b9d484fd091c2ade10e77948f 2024-11-10T09:17:41,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8bc4d25e8b774484b4d915a257ac62f9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/8bc4d25e8b774484b4d915a257ac62f9 2024-11-10T09:17:41,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f173fca5ad144eb2a9d2da66952bab6a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/f173fca5ad144eb2a9d2da66952bab6a 2024-11-10T09:17:41,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2eae9bb94f7c429bb804f41b0c25bcf4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/2eae9bb94f7c429bb804f41b0c25bcf4 2024-11-10T09:17:41,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/fc6fba309f6540dcb8748b51f053b55f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8a16aab773f04d088c19d2bff8cb9275, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/96194392d1f1430c86bdf432d5a32691, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e7a01e55a6a45d69699a351e5b0237c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/300d024255af45d686ccdef3c4866c81, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4fbf4bc9ab994efba74f002e48a9b4f7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/da1d2349d66b4f8784d7e68d43304aed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/28300e6984a84443beb0164832613a9c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c818e7e2b0fa4781ba8383d4be1b9b87, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8166e540441148cf911914ae553d680f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c5f3937a46e54d058c4ef2e2bb36c1df, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/b48419d6b3244a3891db45bb094039a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/3cf1a7c59eaf47e98e10a347c840f211, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/bc05b45141b447bd9e8035da608c6329, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/16b5ad76f8dd455f80f853779af6c613, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/e53943c948194fb48c45ccdf221064d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7929961c401d4de099430d62c3b83512, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/eb0f1c9a9f114e31a3e5018e10f1bcc1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/6ab846a13f8f41dbb46e5820be294bb2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/994da08b25124ffd8799c65e2d2ea0e1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/9b39cd11c8bf489284bd2a2125136095, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/88a1e2ad512746059133f9c8de74a6b5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c8c4f06e39ae40139dcb4d3dc9908649, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7d2dc73bf6da467fb62359e16de97c02, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7cf9ad583efd465a9014bf10a1960ab9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4161c3f069c3480a897178b755240040, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/de4fc2e2c0e24453af277cb5c9444c71, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e054bef1f124c5da34182c29cb9af0b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/58de35e00d624edbaf4bef2eb86c554c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/163243db2d014d058a7090e7050d8b7d] to archive 2024-11-10T09:17:41,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:17:41,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/fc6fba309f6540dcb8748b51f053b55f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/fc6fba309f6540dcb8748b51f053b55f 2024-11-10T09:17:41,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8a16aab773f04d088c19d2bff8cb9275 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8a16aab773f04d088c19d2bff8cb9275 2024-11-10T09:17:41,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/96194392d1f1430c86bdf432d5a32691 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/96194392d1f1430c86bdf432d5a32691 2024-11-10T09:17:41,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e7a01e55a6a45d69699a351e5b0237c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e7a01e55a6a45d69699a351e5b0237c 2024-11-10T09:17:41,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/300d024255af45d686ccdef3c4866c81 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/300d024255af45d686ccdef3c4866c81 2024-11-10T09:17:41,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4fbf4bc9ab994efba74f002e48a9b4f7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4fbf4bc9ab994efba74f002e48a9b4f7 2024-11-10T09:17:41,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/da1d2349d66b4f8784d7e68d43304aed to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/da1d2349d66b4f8784d7e68d43304aed 2024-11-10T09:17:41,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/28300e6984a84443beb0164832613a9c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/28300e6984a84443beb0164832613a9c 2024-11-10T09:17:41,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c818e7e2b0fa4781ba8383d4be1b9b87 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c818e7e2b0fa4781ba8383d4be1b9b87 2024-11-10T09:17:41,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8166e540441148cf911914ae553d680f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8166e540441148cf911914ae553d680f 2024-11-10T09:17:41,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c5f3937a46e54d058c4ef2e2bb36c1df to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c5f3937a46e54d058c4ef2e2bb36c1df 2024-11-10T09:17:41,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/b48419d6b3244a3891db45bb094039a7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/b48419d6b3244a3891db45bb094039a7 2024-11-10T09:17:41,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/3cf1a7c59eaf47e98e10a347c840f211 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/3cf1a7c59eaf47e98e10a347c840f211 2024-11-10T09:17:41,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/bc05b45141b447bd9e8035da608c6329 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/bc05b45141b447bd9e8035da608c6329 2024-11-10T09:17:41,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/16b5ad76f8dd455f80f853779af6c613 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/16b5ad76f8dd455f80f853779af6c613 2024-11-10T09:17:41,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/e53943c948194fb48c45ccdf221064d7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/e53943c948194fb48c45ccdf221064d7 2024-11-10T09:17:41,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7929961c401d4de099430d62c3b83512 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7929961c401d4de099430d62c3b83512 2024-11-10T09:17:41,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/eb0f1c9a9f114e31a3e5018e10f1bcc1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/eb0f1c9a9f114e31a3e5018e10f1bcc1 2024-11-10T09:17:41,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/6ab846a13f8f41dbb46e5820be294bb2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/6ab846a13f8f41dbb46e5820be294bb2 2024-11-10T09:17:41,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/994da08b25124ffd8799c65e2d2ea0e1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/994da08b25124ffd8799c65e2d2ea0e1 2024-11-10T09:17:41,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/9b39cd11c8bf489284bd2a2125136095 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/9b39cd11c8bf489284bd2a2125136095 2024-11-10T09:17:41,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/88a1e2ad512746059133f9c8de74a6b5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/88a1e2ad512746059133f9c8de74a6b5 2024-11-10T09:17:41,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c8c4f06e39ae40139dcb4d3dc9908649 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/c8c4f06e39ae40139dcb4d3dc9908649 2024-11-10T09:17:41,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7d2dc73bf6da467fb62359e16de97c02 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7d2dc73bf6da467fb62359e16de97c02 2024-11-10T09:17:41,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7cf9ad583efd465a9014bf10a1960ab9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/7cf9ad583efd465a9014bf10a1960ab9 2024-11-10T09:17:41,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4161c3f069c3480a897178b755240040 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4161c3f069c3480a897178b755240040 2024-11-10T09:17:41,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/de4fc2e2c0e24453af277cb5c9444c71 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/de4fc2e2c0e24453af277cb5c9444c71 2024-11-10T09:17:41,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e054bef1f124c5da34182c29cb9af0b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/8e054bef1f124c5da34182c29cb9af0b 2024-11-10T09:17:41,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/58de35e00d624edbaf4bef2eb86c554c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/58de35e00d624edbaf4bef2eb86c554c 2024-11-10T09:17:41,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/163243db2d014d058a7090e7050d8b7d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/163243db2d014d058a7090e7050d8b7d 2024-11-10T09:17:41,245 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/recovered.edits/442.seqid, newMaxSeqId=442, maxSeqId=1 2024-11-10T09:17:41,246 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. 2024-11-10T09:17:41,246 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for b031bdc33b2db2f1f99a0ca67fb65c56: Waiting for close lock at 1731230259899Running coprocessor pre-close hooks at 1731230259899Disabling compacts and flushes for region at 1731230259899Disabling writes for close at 1731230259899Obtaining lock to block concurrent updates at 1731230259899Preparing flush snapshotting stores in b031bdc33b2db2f1f99a0ca67fb65c56 at 1731230259899Finished memstore snapshotting TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731230259899Flushing stores of TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56. at 1731230259900 (+1 ms)Flushing b031bdc33b2db2f1f99a0ca67fb65c56/A: creating writer at 1731230259900Flushing b031bdc33b2db2f1f99a0ca67fb65c56/A: appending metadata at 1731230259903 (+3 ms)Flushing b031bdc33b2db2f1f99a0ca67fb65c56/A: closing flushed file at 1731230259903Flushing b031bdc33b2db2f1f99a0ca67fb65c56/B: creating writer at 1731230260312 (+409 ms)Flushing b031bdc33b2db2f1f99a0ca67fb65c56/B: appending metadata at 1731230260315 (+3 ms)Flushing b031bdc33b2db2f1f99a0ca67fb65c56/B: closing flushed file at 1731230260315Flushing b031bdc33b2db2f1f99a0ca67fb65c56/C: creating writer at 1731230260725 (+410 ms)Flushing b031bdc33b2db2f1f99a0ca67fb65c56/C: appending metadata at 1731230260729 (+4 ms)Flushing b031bdc33b2db2f1f99a0ca67fb65c56/C: closing flushed file at 1731230260729Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73351e89: reopening flushed file at 1731230261144 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51027411: reopening flushed file at 1731230261148 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5afd1984: reopening flushed file at 1731230261152 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b031bdc33b2db2f1f99a0ca67fb65c56 in 1256ms, sequenceid=439, compaction requested=true at 1731230261155 (+3 ms)Writing region close event to WAL at 1731230261242 (+87 ms)Running coprocessor post-close hooks at 1731230261246 (+4 ms)Closed at 1731230261246 2024-11-10T09:17:41,248 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:41,248 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b031bdc33b2db2f1f99a0ca67fb65c56, regionState=CLOSED 2024-11-10T09:17:41,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:17:41,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T09:17:41,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure b031bdc33b2db2f1f99a0ca67fb65c56, server=802346a334f5,38513,1731230168538 in 1.5040 sec 2024-11-10T09:17:41,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-10T09:17:41,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b031bdc33b2db2f1f99a0ca67fb65c56, UNASSIGN in 1.5110 sec 2024-11-10T09:17:41,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T09:17:41,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-10T09:17:41,258 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230261258"}]},"ts":"1731230261258"} 2024-11-10T09:17:41,259 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:17:41,259 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:17:41,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5270 sec 2024-11-10T09:17:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:17:41,888 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:41,888 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:17:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:41,890 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:41,890 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T09:17:41,892 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:41,895 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/recovered.edits] 2024-11-10T09:17:41,897 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/0ac88ad9bef14bda81eda4eb23fc71f3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/0ac88ad9bef14bda81eda4eb23fc71f3 2024-11-10T09:17:41,898 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/239b5c9daa7d4bb297dfb3fa32ee6edf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/239b5c9daa7d4bb297dfb3fa32ee6edf 2024-11-10T09:17:41,899 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/efb2e35120544b42846024311a45ff0c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/A/efb2e35120544b42846024311a45ff0c 2024-11-10T09:17:41,900 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/26b06ae3388f4e8980fbd71855f1249f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/26b06ae3388f4e8980fbd71855f1249f 2024-11-10T09:17:41,901 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/3c0cbe1426b748969197067acbd2a2bd to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/3c0cbe1426b748969197067acbd2a2bd 2024-11-10T09:17:41,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b2586505f8264348b673748b52e35ee3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/B/b2586505f8264348b673748b52e35ee3 2024-11-10T09:17:41,905 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4e6a83c579bb46e0ae527a26f3d19ff3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/4e6a83c579bb46e0ae527a26f3d19ff3 2024-11-10T09:17:41,906 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/64f3d7239cbc41a9ac3fc507819aa3f2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/64f3d7239cbc41a9ac3fc507819aa3f2 2024-11-10T09:17:41,907 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/f2f8e8b096c149f3a82f1a72390ce61f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/C/f2f8e8b096c149f3a82f1a72390ce61f 2024-11-10T09:17:41,910 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/recovered.edits/442.seqid to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56/recovered.edits/442.seqid 2024-11-10T09:17:41,910 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/b031bdc33b2db2f1f99a0ca67fb65c56 2024-11-10T09:17:41,910 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:17:41,913 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:41,916 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:17:41,919 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:17:41,920 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:41,920 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:17:41,920 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230261920"}]},"ts":"9223372036854775807"} 2024-11-10T09:17:41,924 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:17:41,924 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b031bdc33b2db2f1f99a0ca67fb65c56, NAME => 'TestAcidGuarantees,,1731230228263.b031bdc33b2db2f1f99a0ca67fb65c56.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:17:41,924 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:17:41,924 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230261924"}]},"ts":"9223372036854775807"} 2024-11-10T09:17:41,926 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:17:41,927 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:41,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-10T09:17:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T09:17:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:17:41,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:42,006 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 252), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=737 (was 826), ProcessCount=11 (was 11), AvailableMemoryMB=1191 (was 1857) 2024-11-10T09:17:42,013 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=737, ProcessCount=11, AvailableMemoryMB=1191 2024-11-10T09:17:42,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:17:42,015 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:17:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:42,017 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:17:42,018 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:42,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-10T09:17:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T09:17:42,019 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:17:42,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742171_1347 (size=1061) 2024-11-10T09:17:42,045 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:17:42,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742172_1348 (size=53) 2024-11-10T09:17:42,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:17:42,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 32f231a2c762c9b2ac161a81d8fdd9d4, disabling compactions & flushes 2024-11-10T09:17:42,064 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. after waiting 0 ms 2024-11-10T09:17:42,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,064 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 32f231a2c762c9b2ac161a81d8fdd9d4: Waiting for close lock at 1731230262064Disabling compacts and flushes for region at 1731230262064Disabling writes for close at 1731230262064Writing region close event to WAL at 1731230262064Closed at 1731230262064 2024-11-10T09:17:42,066 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:17:42,066 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230262066"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230262066"}]},"ts":"1731230262066"} 2024-11-10T09:17:42,070 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:17:42,073 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:17:42,074 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230262074"}]},"ts":"1731230262074"} 2024-11-10T09:17:42,077 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:17:42,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, ASSIGN}] 2024-11-10T09:17:42,079 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, ASSIGN 2024-11-10T09:17:42,080 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, ASSIGN; state=OFFLINE, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=false 2024-11-10T09:17:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T09:17:42,230 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:42,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, ASSIGN because future has completed 2024-11-10T09:17:42,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:17:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T09:17:42,391 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,392 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:17:42,392 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,392 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:17:42,392 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,392 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,394 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,396 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:42,396 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32f231a2c762c9b2ac161a81d8fdd9d4 columnFamilyName A 2024-11-10T09:17:42,396 DEBUG [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:42,397 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(327): Store=32f231a2c762c9b2ac161a81d8fdd9d4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:42,397 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,398 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:42,398 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32f231a2c762c9b2ac161a81d8fdd9d4 columnFamilyName B 2024-11-10T09:17:42,398 DEBUG [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:42,399 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(327): Store=32f231a2c762c9b2ac161a81d8fdd9d4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:42,399 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,400 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:42,400 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32f231a2c762c9b2ac161a81d8fdd9d4 columnFamilyName C 2024-11-10T09:17:42,400 DEBUG [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:42,401 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(327): Store=32f231a2c762c9b2ac161a81d8fdd9d4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:42,401 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,401 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,402 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,402 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,403 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,403 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,403 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:17:42,404 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,406 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:17:42,406 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 32f231a2c762c9b2ac161a81d8fdd9d4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63834237, jitterRate=-0.04879574477672577}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:17:42,406 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:42,407 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 32f231a2c762c9b2ac161a81d8fdd9d4: Running coprocessor pre-open hook at 1731230262392Writing region info on filesystem at 1731230262392Initializing all the Stores at 1731230262393 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230262393Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230262394 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230262394Cleaning up temporary data from old regions at 1731230262403 (+9 ms)Running coprocessor post-open hooks at 1731230262406 (+3 ms)Region opened successfully at 1731230262407 (+1 ms) 2024-11-10T09:17:42,407 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., pid=89, masterSystemTime=1731230262388 2024-11-10T09:17:42,409 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,409 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:42,410 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:42,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:17:42,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-10T09:17:42,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 in 178 msec 2024-11-10T09:17:42,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-10T09:17:42,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, ASSIGN in 338 msec 2024-11-10T09:17:42,417 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:17:42,417 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230262417"}]},"ts":"1731230262417"} 2024-11-10T09:17:42,419 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:17:42,420 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:17:42,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 405 msec 2024-11-10T09:17:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T09:17:42,648 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:42,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11851421, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:42,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:42,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:42,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:42,655 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c60ea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:42,656 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:42,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:42,660 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20895db9 2024-11-10T09:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:17:42,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:17:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:17:42,664 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:17:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T09:17:42,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742173_1349 (size=1097) 2024-11-10T09:17:43,076 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T09:17:43,076 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T09:17:43,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:17:43,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, REOPEN/MOVE}] 2024-11-10T09:17:43,086 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, REOPEN/MOVE 2024-11-10T09:17:43,086 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, REOPEN/MOVE because future has completed 2024-11-10T09:17:43,088 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:17:43,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:17:43,240 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,240 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:17:43,240 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 32f231a2c762c9b2ac161a81d8fdd9d4, disabling compactions & flushes 2024-11-10T09:17:43,240 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,240 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,240 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. after waiting 0 ms 2024-11-10T09:17:43,240 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,244 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T09:17:43,245 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,245 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 32f231a2c762c9b2ac161a81d8fdd9d4: Waiting for close lock at 1731230263240Running coprocessor pre-close hooks at 1731230263240Disabling compacts and flushes for region at 1731230263240Disabling writes for close at 1731230263240Writing region close event to WAL at 1731230263241 (+1 ms)Running coprocessor post-close hooks at 1731230263245 (+4 ms)Closed at 1731230263245 2024-11-10T09:17:43,245 WARN [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 32f231a2c762c9b2ac161a81d8fdd9d4 to self. 2024-11-10T09:17:43,246 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=CLOSED 2024-11-10T09:17:43,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:17:43,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T09:17:43,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 in 161 msec 2024-11-10T09:17:43,251 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, REOPEN/MOVE; state=CLOSED, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=true 2024-11-10T09:17:43,402 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, REOPEN/MOVE because future has completed 2024-11-10T09:17:43,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:17:43,561 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,561 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:17:43,561 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,562 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:17:43,562 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,562 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,566 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,567 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:43,567 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32f231a2c762c9b2ac161a81d8fdd9d4 columnFamilyName A 2024-11-10T09:17:43,568 DEBUG [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:43,569 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(327): Store=32f231a2c762c9b2ac161a81d8fdd9d4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:43,569 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,569 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:43,570 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32f231a2c762c9b2ac161a81d8fdd9d4 columnFamilyName B 2024-11-10T09:17:43,570 DEBUG [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:43,570 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(327): Store=32f231a2c762c9b2ac161a81d8fdd9d4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:43,570 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,570 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:17:43,571 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32f231a2c762c9b2ac161a81d8fdd9d4 columnFamilyName C 2024-11-10T09:17:43,571 DEBUG [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:43,571 INFO [StoreOpener-32f231a2c762c9b2ac161a81d8fdd9d4-1 {}] regionserver.HStore(327): Store=32f231a2c762c9b2ac161a81d8fdd9d4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:17:43,571 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,571 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,572 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,573 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,574 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,574 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,574 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:17:43,575 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,576 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 32f231a2c762c9b2ac161a81d8fdd9d4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65699894, jitterRate=-0.020995289087295532}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:17:43,576 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,577 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 32f231a2c762c9b2ac161a81d8fdd9d4: Running coprocessor pre-open hook at 1731230263562Writing region info on filesystem at 1731230263562Initializing all the Stores at 1731230263562Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230263563 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230263566 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230263566Cleaning up temporary data from old regions at 1731230263574 (+8 ms)Running coprocessor post-open hooks at 1731230263576 (+2 ms)Region opened successfully at 1731230263577 (+1 ms) 2024-11-10T09:17:43,577 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., pid=94, masterSystemTime=1731230263558 2024-11-10T09:17:43,579 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,579 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,580 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=OPEN, openSeqNum=5, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:17:43,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-10T09:17:43,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 in 177 msec 2024-11-10T09:17:43,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-10T09:17:43,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, REOPEN/MOVE in 499 msec 2024-11-10T09:17:43,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T09:17:43,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-10T09:17:43,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-11-10T09:17:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:17:43,591 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:43,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335bd9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c094db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e060b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549ff36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402982fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed3beef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1495b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477c0a2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbee311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a145a37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5960be21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,606 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1f8079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7933e7cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,608 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d67749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,609 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1763b9b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b124c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,611 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d51450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,612 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c60735c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd5b9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,616 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed175f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c93281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,617 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bac148f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4102fd43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5938121e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,621 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b3fad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13744eee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40993d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:17:43,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:17:43,625 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:17:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:17:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:17:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb36129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:17:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:17:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:17:43,627 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:17:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46faf1e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:17:43,628 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,628 DEBUG [Thread-1587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,629 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,629 DEBUG [Thread-1600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,630 DEBUG [Thread-1591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,637 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,637 DEBUG [Thread-1589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,638 DEBUG [Thread-1593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,644 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:43,647 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:17:43,648 DEBUG [Thread-1596 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:17:43,648 DEBUG [Thread-1596 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:17:43,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:17:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-10T09:17:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:17:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T09:17:43,655 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:43,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:43,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:17:43,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:43,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:43,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:43,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:43,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:43,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:43,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108db0a4a7224f423c830ff28e9915a056_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230263661/Put/seqid=0 2024-11-10T09:17:43,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742174_1350 (size=12154) 2024-11-10T09:17:43,707 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:43,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230263684, max timestamp=1731230263705Num uniques 859; 2024-11-10T09:17:43,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:43,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230263684, max timestamp=1731230263705Num uniques 859; 2024-11-10T09:17:43,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:43,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230263684, max timestamp=1731230263705Num uniques 859; 2024-11-10T09:17:43,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:43,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108db0a4a7224f423c830ff28e9915a056_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108db0a4a7224f423c830ff28e9915a056_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:43,721 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/2e2f81dce0a44db88e028073338ed7f9, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:43,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/2e2f81dce0a44db88e028073338ed7f9 is 175, key is test_row_0/A:col10/1731230263661/Put/seqid=0 2024-11-10T09:17:43,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742175_1351 (size=30955) 2024-11-10T09:17:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T09:17:43,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230323756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230323778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230323778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230323778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230323780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T09:17:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:43,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230323897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230323898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230323900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230323900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:43,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T09:17:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:43,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:43,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T09:17:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230324108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230324108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T09:17:44,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:44,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:44,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:44,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:44,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:44,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/2e2f81dce0a44db88e028073338ed7f9 2024-11-10T09:17:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230324119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230324120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230324120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a9dae62afd92415093852a3fa9fff77d is 50, key is test_row_0/B:col10/1731230263661/Put/seqid=0 2024-11-10T09:17:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742176_1352 (size=12001) 2024-11-10T09:17:44,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a9dae62afd92415093852a3fa9fff77d 2024-11-10T09:17:44,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/0985949fd22849c49bc452c7dec4ae64 is 50, key is test_row_0/C:col10/1731230263661/Put/seqid=0 2024-11-10T09:17:44,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742177_1353 (size=12001) 2024-11-10T09:17:44,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/0985949fd22849c49bc452c7dec4ae64 2024-11-10T09:17:44,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/2e2f81dce0a44db88e028073338ed7f9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9 2024-11-10T09:17:44,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9, entries=150, sequenceid=14, filesize=30.2 K 2024-11-10T09:17:44,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a9dae62afd92415093852a3fa9fff77d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a9dae62afd92415093852a3fa9fff77d 2024-11-10T09:17:44,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a9dae62afd92415093852a3fa9fff77d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T09:17:44,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/0985949fd22849c49bc452c7dec4ae64 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0985949fd22849c49bc452c7dec4ae64 2024-11-10T09:17:44,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0985949fd22849c49bc452c7dec4ae64, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T09:17:44,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T09:17:44,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 612ms, sequenceid=14, compaction requested=false 2024-11-10T09:17:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:44,274 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T09:17:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:44,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:44,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T09:17:44,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:44,427 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:44,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d9777150ace24bcbae6ef16a096f37d2_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230263778/Put/seqid=0 2024-11-10T09:17:44,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742178_1354 (size=12154) 2024-11-10T09:17:44,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:44,441 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d9777150ace24bcbae6ef16a096f37d2_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9777150ace24bcbae6ef16a096f37d2_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:44,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c1f1123573144c8f99463229edbcfc34, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c1f1123573144c8f99463229edbcfc34 is 175, key is test_row_0/A:col10/1731230263778/Put/seqid=0 2024-11-10T09:17:44,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742179_1355 (size=30955) 2024-11-10T09:17:44,447 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c1f1123573144c8f99463229edbcfc34 2024-11-10T09:17:44,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230324440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230324445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/f646825aef78409c8981fdf79885a54c is 50, key is test_row_0/B:col10/1731230263778/Put/seqid=0 2024-11-10T09:17:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230324446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230324446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230324448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742180_1356 (size=12001) 2024-11-10T09:17:44,471 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/f646825aef78409c8981fdf79885a54c 2024-11-10T09:17:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/b9e2639490724b0ab2dcab2cc970c92a is 50, key is test_row_0/C:col10/1731230263778/Put/seqid=0 2024-11-10T09:17:44,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742181_1357 (size=12001) 2024-11-10T09:17:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230324557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230324557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230324567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230324567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230324577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230324767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230324777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230324777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230324777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T09:17:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230324797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:44,886 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/b9e2639490724b0ab2dcab2cc970c92a 2024-11-10T09:17:44,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c1f1123573144c8f99463229edbcfc34 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34 2024-11-10T09:17:44,894 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34, entries=150, sequenceid=49, filesize=30.2 K 2024-11-10T09:17:44,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/f646825aef78409c8981fdf79885a54c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f646825aef78409c8981fdf79885a54c 2024-11-10T09:17:44,898 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f646825aef78409c8981fdf79885a54c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:17:44,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/b9e2639490724b0ab2dcab2cc970c92a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/b9e2639490724b0ab2dcab2cc970c92a 2024-11-10T09:17:44,902 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/b9e2639490724b0ab2dcab2cc970c92a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:17:44,903 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 476ms, sequenceid=49, compaction requested=false 2024-11-10T09:17:44,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:44,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:44,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-10T09:17:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-10T09:17:44,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-10T09:17:44,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2480 sec 2024-11-10T09:17:44,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.2590 sec 2024-11-10T09:17:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:45,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:45,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bd8b1417569b4c07b8c3eab9c8ced5a2_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:45,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742182_1358 (size=14594) 2024-11-10T09:17:45,108 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:45,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bd8b1417569b4c07b8c3eab9c8ced5a2_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bd8b1417569b4c07b8c3eab9c8ced5a2_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:45,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/5ab71ef88fe746458f4dade5ca1677a2, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:45,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/5ab71ef88fe746458f4dade5ca1677a2 is 175, key is test_row_0/A:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:45,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742183_1359 (size=39549) 2024-11-10T09:17:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230265141, max timestamp=1731230265160Num uniques 859; 2024-11-10T09:17:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230265141, max timestamp=1731230265160Num uniques 859; 2024-11-10T09:17:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230265141, max timestamp=1731230265153Num uniques 859; 2024-11-10T09:17:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:17:45,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230325216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230325217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230325218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230325218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230325222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230325337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230325337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230325338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230325338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230325338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/5ab71ef88fe746458f4dade5ca1677a2 2024-11-10T09:17:45,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/b41861a665234d0bbc17c7e8ae0c7ecb is 50, key is test_row_0/B:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230325547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230325557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742184_1360 (size=12001) 2024-11-10T09:17:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230325558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230325558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230325558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T09:17:45,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:45,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T09:17:45,800 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:45,801 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:45,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230325867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230325877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230325877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230325878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230325878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:45,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:45,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:45,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:45,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:45,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:45,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:45,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/b41861a665234d0bbc17c7e8ae0c7ecb 2024-11-10T09:17:45,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/987e4922486f4aa7baa7474f31a0342b is 50, key is test_row_0/C:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:45,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742185_1361 (size=12001) 2024-11-10T09:17:46,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:46,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:46,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:46,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/987e4922486f4aa7baa7474f31a0342b 2024-11-10T09:17:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/5ab71ef88fe746458f4dade5ca1677a2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2 2024-11-10T09:17:46,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2, entries=200, sequenceid=60, filesize=38.6 K 2024-11-10T09:17:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/b41861a665234d0bbc17c7e8ae0c7ecb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/b41861a665234d0bbc17c7e8ae0c7ecb 2024-11-10T09:17:46,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/b41861a665234d0bbc17c7e8ae0c7ecb, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T09:17:46,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/987e4922486f4aa7baa7474f31a0342b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/987e4922486f4aa7baa7474f31a0342b 2024-11-10T09:17:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230326387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/987e4922486f4aa7baa7474f31a0342b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T09:17:46,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1312ms, sequenceid=60, compaction requested=true 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:46,403 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:17:46,406 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:46,406 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:46,406 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:46,406 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:17:46,406 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:17:46,407 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,407 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,407 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=99.1 K 2024-11-10T09:17:46,407 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a9dae62afd92415093852a3fa9fff77d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f646825aef78409c8981fdf79885a54c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/b41861a665234d0bbc17c7e8ae0c7ecb] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=35.2 K 2024-11-10T09:17:46,407 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,407 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2] 2024-11-10T09:17:46,407 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting a9dae62afd92415093852a3fa9fff77d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731230263644 2024-11-10T09:17:46,407 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e2f81dce0a44db88e028073338ed7f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731230263644 2024-11-10T09:17:46,407 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f1123573144c8f99463229edbcfc34, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230263742 2024-11-10T09:17:46,407 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f646825aef78409c8981fdf79885a54c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230263742 2024-11-10T09:17:46,408 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b41861a665234d0bbc17c7e8ae0c7ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230264445 2024-11-10T09:17:46,408 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab71ef88fe746458f4dade5ca1677a2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230264444 2024-11-10T09:17:46,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T09:17:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:46,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:46,437 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:46,439 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:46,440 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/f1fb6f7ce46a48fea8137c475411db97 is 50, key is test_row_0/B:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:46,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cf87522e25d048189c83822fd99f44bd_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230265215/Put/seqid=0 2024-11-10T09:17:46,446 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ddecc0089ded447caef8ecdedd5c6175_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:46,448 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ddecc0089ded447caef8ecdedd5c6175_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:46,448 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ddecc0089ded447caef8ecdedd5c6175_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230326432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230326432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230326432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230326438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742187_1363 (size=14594) 2024-11-10T09:17:46,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742186_1362 (size=12104) 2024-11-10T09:17:46,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742188_1364 (size=4469) 2024-11-10T09:17:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230326557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230326557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230326557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230326567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,782 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:17:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230326777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230326777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230326777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230326778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:46,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:46,873 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:46,874 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/6b22478a855442bdb1fad2fa904986b1 is 175, key is test_row_0/A:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:46,876 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cf87522e25d048189c83822fd99f44bd_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cf87522e25d048189c83822fd99f44bd_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:46,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0f1b349bf0084c2596a2981cfb22e8d2, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:46,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0f1b349bf0084c2596a2981cfb22e8d2 is 175, key is test_row_0/A:col10/1731230265215/Put/seqid=0 2024-11-10T09:17:46,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,882 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/f1fb6f7ce46a48fea8137c475411db97 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f1fb6f7ce46a48fea8137c475411db97 2024-11-10T09:17:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:46,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742189_1365 (size=31058) 2024-11-10T09:17:46,891 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into f1fb6f7ce46a48fea8137c475411db97(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:46,891 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:46,891 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230266403; duration=0sec 2024-11-10T09:17:46,892 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:46,892 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:17:46,892 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:46,893 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:46,894 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:17:46,894 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:46,894 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0985949fd22849c49bc452c7dec4ae64, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/b9e2639490724b0ab2dcab2cc970c92a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/987e4922486f4aa7baa7474f31a0342b] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=35.2 K 2024-11-10T09:17:46,894 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0985949fd22849c49bc452c7dec4ae64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731230263644 2024-11-10T09:17:46,895 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e2639490724b0ab2dcab2cc970c92a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230263742 2024-11-10T09:17:46,895 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/6b22478a855442bdb1fad2fa904986b1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/6b22478a855442bdb1fad2fa904986b1 2024-11-10T09:17:46,895 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 987e4922486f4aa7baa7474f31a0342b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230264445 2024-11-10T09:17:46,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742190_1366 (size=39549) 2024-11-10T09:17:46,905 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into 6b22478a855442bdb1fad2fa904986b1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:46,905 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:46,905 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230266403; duration=0sec 2024-11-10T09:17:46,905 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:46,905 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:17:46,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0f1b349bf0084c2596a2981cfb22e8d2 2024-11-10T09:17:46,906 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:46,906 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/d533605489614e30be595e50e92749d7 is 50, key is test_row_0/C:col10/1731230264445/Put/seqid=0 2024-11-10T09:17:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:46,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742191_1367 (size=12104) 2024-11-10T09:17:46,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/55c0ea81bf44437b875ecdd92d3b7126 is 50, key is test_row_0/B:col10/1731230265215/Put/seqid=0 2024-11-10T09:17:46,943 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/d533605489614e30be595e50e92749d7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d533605489614e30be595e50e92749d7 2024-11-10T09:17:46,949 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into d533605489614e30be595e50e92749d7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:46,949 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:46,949 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230266403; duration=0sec 2024-11-10T09:17:46,949 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:46,949 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:17:46,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742192_1368 (size=12001) 2024-11-10T09:17:46,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/55c0ea81bf44437b875ecdd92d3b7126 2024-11-10T09:17:46,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/4642d9a3eba948e5baad6cd1cba65e56 is 50, key is test_row_0/C:col10/1731230265215/Put/seqid=0 2024-11-10T09:17:47,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742193_1369 (size=12001) 2024-11-10T09:17:47,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230327107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230327107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230327107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230327107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:47,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:47,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230327407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/4642d9a3eba948e5baad6cd1cba65e56 2024-11-10T09:17:47,422 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 32f231a2c762c9b2ac161a81d8fdd9d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0f1b349bf0084c2596a2981cfb22e8d2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2 2024-11-10T09:17:47,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2, entries=200, sequenceid=97, filesize=38.6 K 2024-11-10T09:17:47,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/55c0ea81bf44437b875ecdd92d3b7126 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/55c0ea81bf44437b875ecdd92d3b7126 2024-11-10T09:17:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/55c0ea81bf44437b875ecdd92d3b7126, entries=150, sequenceid=97, filesize=11.7 K 2024-11-10T09:17:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/4642d9a3eba948e5baad6cd1cba65e56 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4642d9a3eba948e5baad6cd1cba65e56 2024-11-10T09:17:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4642d9a3eba948e5baad6cd1cba65e56, entries=150, sequenceid=97, filesize=11.7 K 2024-11-10T09:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1055ms, sequenceid=97, compaction requested=false 2024-11-10T09:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T09:17:47,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,498 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:17:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101374dcf4e96747759a97cde610102b2b_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230266412/Put/seqid=0 2024-11-10T09:17:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742194_1370 (size=9714) 2024-11-10T09:17:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,525 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101374dcf4e96747759a97cde610102b2b_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101374dcf4e96747759a97cde610102b2b_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/735ee2424ad642b1980c5703c66536ac, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/735ee2424ad642b1980c5703c66536ac is 175, key is test_row_0/A:col10/1731230266412/Put/seqid=0 2024-11-10T09:17:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742195_1371 (size=22361) 2024-11-10T09:17:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/735ee2424ad642b1980c5703c66536ac 2024-11-10T09:17:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a832e81e3c4241cda1d87c9653bf3785 is 50, key is test_row_0/B:col10/1731230266412/Put/seqid=0 2024-11-10T09:17:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742196_1372 (size=9657) 2024-11-10T09:17:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:47,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:47,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731230267667, max timestamp=1731230267719Num uniques 859; 2024-11-10T09:17:47,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731230267667, max timestamp=1731230267719Num uniques 859; 2024-11-10T09:17:47,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731230267667, max timestamp=1731230267719Num uniques 859; 2024-11-10T09:17:47,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T09:17:47,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T09:17:47,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T09:17:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230327754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230327760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230327766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230327767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230327877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230327877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230327897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230327897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:48,015 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a832e81e3c4241cda1d87c9653bf3785 2024-11-10T09:17:48,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/cf6973df617b4fb4bd0ce241c706dce0 is 50, key is test_row_0/C:col10/1731230266412/Put/seqid=0 2024-11-10T09:17:48,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742197_1373 (size=9657) 2024-11-10T09:17:48,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230328099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230328101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230328117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230328117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230328419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230328427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230328427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230328427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,504 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/cf6973df617b4fb4bd0ce241c706dce0 2024-11-10T09:17:48,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/735ee2424ad642b1980c5703c66536ac as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac 2024-11-10T09:17:48,515 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac, entries=100, sequenceid=110, filesize=21.8 K 2024-11-10T09:17:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a832e81e3c4241cda1d87c9653bf3785 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a832e81e3c4241cda1d87c9653bf3785 2024-11-10T09:17:48,520 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a832e81e3c4241cda1d87c9653bf3785, entries=100, sequenceid=110, filesize=9.4 K 2024-11-10T09:17:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/cf6973df617b4fb4bd0ce241c706dce0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cf6973df617b4fb4bd0ce241c706dce0 2024-11-10T09:17:48,529 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cf6973df617b4fb4bd0ce241c706dce0, entries=100, sequenceid=110, filesize=9.4 K 2024-11-10T09:17:48,530 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1032ms, sequenceid=110, compaction requested=true 2024-11-10T09:17:48,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:48,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:48,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T09:17:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T09:17:48,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T09:17:48,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7310 sec 2024-11-10T09:17:48,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7360 sec 2024-11-10T09:17:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:48,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-10T09:17:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:48,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:48,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:48,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108e6fde44bffe4d7998c8fac46d131e9a_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:48,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742198_1374 (size=14794) 2024-11-10T09:17:48,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230328959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108e6fde44bffe4d7998c8fac46d131e9a_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108e6fde44bffe4d7998c8fac46d131e9a_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:48,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/130a8320022b4b0a96ace8dfe739105e, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:48,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/130a8320022b4b0a96ace8dfe739105e is 175, key is test_row_0/A:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:48,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742199_1375 (size=39749) 2024-11-10T09:17:48,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230328972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230328973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:48,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230328976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230329087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230329098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230329307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230329307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230329307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230329318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,385 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/130a8320022b4b0a96ace8dfe739105e 2024-11-10T09:17:49,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5377855efd4344c2a91a06bd02c4ddc5 is 50, key is test_row_0/B:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:49,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742200_1376 (size=12151) 2024-11-10T09:17:49,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5377855efd4344c2a91a06bd02c4ddc5 2024-11-10T09:17:49,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/07c380b6f7a2454fb4cecb4aeeb4d960 is 50, key is test_row_0/C:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:49,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742201_1377 (size=12151) 2024-11-10T09:17:49,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/07c380b6f7a2454fb4cecb4aeeb4d960 2024-11-10T09:17:49,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230329440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,456 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 32f231a2c762c9b2ac161a81d8fdd9d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:49,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/130a8320022b4b0a96ace8dfe739105e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e 2024-11-10T09:17:49,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e, entries=200, sequenceid=146, filesize=38.8 K 2024-11-10T09:17:49,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5377855efd4344c2a91a06bd02c4ddc5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5377855efd4344c2a91a06bd02c4ddc5 2024-11-10T09:17:49,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5377855efd4344c2a91a06bd02c4ddc5, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T09:17:49,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/07c380b6f7a2454fb4cecb4aeeb4d960 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/07c380b6f7a2454fb4cecb4aeeb4d960 2024-11-10T09:17:49,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/07c380b6f7a2454fb4cecb4aeeb4d960, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T09:17:49,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 542ms, sequenceid=146, compaction requested=true 2024-11-10T09:17:49,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:49,492 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:17:49,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:49,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:49,492 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:17:49,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:49,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:49,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:49,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:49,497 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:17:49,497 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:17:49,498 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:49,498 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f1fb6f7ce46a48fea8137c475411db97, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/55c0ea81bf44437b875ecdd92d3b7126, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a832e81e3c4241cda1d87c9653bf3785, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5377855efd4344c2a91a06bd02c4ddc5] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=44.8 K 2024-11-10T09:17:49,498 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:17:49,498 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:17:49,498 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:49,498 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/6b22478a855442bdb1fad2fa904986b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=129.6 K 2024-11-10T09:17:49,498 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:49,498 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/6b22478a855442bdb1fad2fa904986b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e] 2024-11-10T09:17:49,498 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b22478a855442bdb1fad2fa904986b1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230264445 2024-11-10T09:17:49,498 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fb6f7ce46a48fea8137c475411db97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230264445 2024-11-10T09:17:49,499 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1b349bf0084c2596a2981cfb22e8d2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731230265215 2024-11-10T09:17:49,499 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 55c0ea81bf44437b875ecdd92d3b7126, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731230265215 2024-11-10T09:17:49,499 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735ee2424ad642b1980c5703c66536ac, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731230266412 2024-11-10T09:17:49,499 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting a832e81e3c4241cda1d87c9653bf3785, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731230266412 2024-11-10T09:17:49,500 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130a8320022b4b0a96ace8dfe739105e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731230267752 2024-11-10T09:17:49,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5377855efd4344c2a91a06bd02c4ddc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731230267752 2024-11-10T09:17:49,511 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:49,511 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/74e7895acb9e402eae08451ab0833c34 is 50, key is test_row_0/B:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:49,533 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:49,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742202_1378 (size=12391) 2024-11-10T09:17:49,569 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c75cd7c4817f489ca5fcfdfdaf0f70c9_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:49,571 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c75cd7c4817f489ca5fcfdfdaf0f70c9_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:49,571 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c75cd7c4817f489ca5fcfdfdaf0f70c9_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742203_1379 (size=4469) 2024-11-10T09:17:49,608 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#322 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:49,609 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/b3384916ea534f3398dfcc108716f705 is 175, key is test_row_0/A:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:49,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:17:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742204_1380 (size=31345) 2024-11-10T09:17:49,659 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/b3384916ea534f3398dfcc108716f705 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/b3384916ea534f3398dfcc108716f705 2024-11-10T09:17:49,664 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into b3384916ea534f3398dfcc108716f705(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:49,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:49,664 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=12, startTime=1731230269492; duration=0sec 2024-11-10T09:17:49,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:49,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:17:49,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:17:49,668 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:17:49,668 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:17:49,668 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:49,668 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d533605489614e30be595e50e92749d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4642d9a3eba948e5baad6cd1cba65e56, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cf6973df617b4fb4bd0ce241c706dce0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/07c380b6f7a2454fb4cecb4aeeb4d960] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=44.8 K 2024-11-10T09:17:49,669 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d533605489614e30be595e50e92749d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230264445 2024-11-10T09:17:49,677 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4642d9a3eba948e5baad6cd1cba65e56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731230265215 2024-11-10T09:17:49,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110957207fe3d2446b9aee6b14944c476db_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230268971/Put/seqid=0 2024-11-10T09:17:49,680 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf6973df617b4fb4bd0ce241c706dce0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731230266412 2024-11-10T09:17:49,687 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07c380b6f7a2454fb4cecb4aeeb4d960, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731230267752 2024-11-10T09:17:49,706 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:49,706 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/2f4adb079ae142c699a969ad68d160d7 is 50, key is test_row_0/C:col10/1731230268947/Put/seqid=0 2024-11-10T09:17:49,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742205_1381 (size=14794) 2024-11-10T09:17:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230269697, max timestamp=1731230269714Num uniques 859; 2024-11-10T09:17:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230269697, max timestamp=1731230269714Num uniques 859; 2024-11-10T09:17:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230269697, max timestamp=1731230269714Num uniques 859; 2024-11-10T09:17:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:49,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742206_1382 (size=12391) 2024-11-10T09:17:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230329801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230329802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230329804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230329810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230329918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230329937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230329937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230329940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:17:49,968 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:49,968 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T09:17:49,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:49,972 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:49,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:49,997 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/74e7895acb9e402eae08451ab0833c34 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/74e7895acb9e402eae08451ab0833c34 2024-11-10T09:17:50,033 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into 74e7895acb9e402eae08451ab0833c34(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:50,033 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:50,033 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=12, startTime=1731230269492; duration=0sec 2024-11-10T09:17:50,034 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:50,034 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:17:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:50,112 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:50,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110957207fe3d2446b9aee6b14944c476db_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110957207fe3d2446b9aee6b14944c476db_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:50,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/471e0dde4bb54619bba562fd7b89de20, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:50,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:50,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:50,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/471e0dde4bb54619bba562fd7b89de20 is 175, key is test_row_0/A:col10/1731230268971/Put/seqid=0 2024-11-10T09:17:50,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,135 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/2f4adb079ae142c699a969ad68d160d7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/2f4adb079ae142c699a969ad68d160d7 2024-11-10T09:17:50,150 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into 2f4adb079ae142c699a969ad68d160d7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:50,150 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:50,150 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=12, startTime=1731230269492; duration=0sec 2024-11-10T09:17:50,150 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:50,150 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:17:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230330141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742207_1383 (size=39749) 2024-11-10T09:17:50,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/471e0dde4bb54619bba562fd7b89de20 2024-11-10T09:17:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230330168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230330168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230330172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/ec0d0eca843143a7bbbf4c6cebb95254 is 50, key is test_row_0/B:col10/1731230268971/Put/seqid=0 2024-11-10T09:17:50,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742208_1384 (size=12151) 2024-11-10T09:17:50,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:50,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:50,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:50,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230330468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230330488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230330488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230330488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:50,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:50,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:50,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/ec0d0eca843143a7bbbf4c6cebb95254 2024-11-10T09:17:50,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0 is 50, key is test_row_0/C:col10/1731230268971/Put/seqid=0 2024-11-10T09:17:50,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742209_1385 (size=12151) 2024-11-10T09:17:50,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0 2024-11-10T09:17:50,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/471e0dde4bb54619bba562fd7b89de20 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20 2024-11-10T09:17:50,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20, entries=200, sequenceid=156, filesize=38.8 K 2024-11-10T09:17:50,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/ec0d0eca843143a7bbbf4c6cebb95254 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ec0d0eca843143a7bbbf4c6cebb95254 2024-11-10T09:17:50,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ec0d0eca843143a7bbbf4c6cebb95254, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T09:17:50,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0 2024-11-10T09:17:50,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T09:17:50,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:50,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:50,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:50,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:50,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1306ms, sequenceid=156, compaction requested=false 2024-11-10T09:17:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:51,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-10T09:17:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:51,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c22209eddef44b6b4def8ec99d26697_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230331035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230331054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230331056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230331058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:51,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742210_1386 (size=14794) 2024-11-10T09:17:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:51,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:51,125 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c22209eddef44b6b4def8ec99d26697_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c22209eddef44b6b4def8ec99d26697_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:51,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/bb9e49de1cec4bb688fd90ea02c5311f, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:51,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/bb9e49de1cec4bb688fd90ea02c5311f is 175, key is test_row_0/A:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:51,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230331176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230331178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230331187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230331188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742211_1387 (size=39749) 2024-11-10T09:17:51,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:51,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:51,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230331398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230331400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230331408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230331418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/bb9e49de1cec4bb688fd90ea02c5311f 2024-11-10T09:17:51,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a5172869928a4cf2967608f01e232cd3 is 50, key is test_row_0/B:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:51,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742212_1388 (size=12151) 2024-11-10T09:17:51,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230331724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230331728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230331729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230331737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:51,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:51,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a5172869928a4cf2967608f01e232cd3 2024-11-10T09:17:52,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/a7cefb22863441909360af0029e8066e is 50, key is test_row_0/C:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:52,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742213_1389 (size=12151) 2024-11-10T09:17:52,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230332248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:52,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230332258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:52,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230332258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230332258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:52,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:52,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:52,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/a7cefb22863441909360af0029e8066e 2024-11-10T09:17:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/bb9e49de1cec4bb688fd90ea02c5311f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f 2024-11-10T09:17:52,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f, entries=200, sequenceid=195, filesize=38.8 K 2024-11-10T09:17:52,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a5172869928a4cf2967608f01e232cd3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a5172869928a4cf2967608f01e232cd3 2024-11-10T09:17:52,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a5172869928a4cf2967608f01e232cd3, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:17:52,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/a7cefb22863441909360af0029e8066e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/a7cefb22863441909360af0029e8066e 2024-11-10T09:17:52,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/a7cefb22863441909360af0029e8066e, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:17:52,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1578ms, sequenceid=195, compaction requested=true 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:52,605 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:52,605 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:52,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:52,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:52,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:17:52,606 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,607 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/74e7895acb9e402eae08451ab0833c34, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ec0d0eca843143a7bbbf4c6cebb95254, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a5172869928a4cf2967608f01e232cd3] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=35.8 K 2024-11-10T09:17:52,607 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 74e7895acb9e402eae08451ab0833c34, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731230267752 2024-11-10T09:17:52,607 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0d0eca843143a7bbbf4c6cebb95254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731230268958 2024-11-10T09:17:52,607 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting a5172869928a4cf2967608f01e232cd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230269779 2024-11-10T09:17:52,608 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:52,608 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:17:52,609 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,609 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/b3384916ea534f3398dfcc108716f705, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=108.2 K 2024-11-10T09:17:52,609 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,609 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/b3384916ea534f3398dfcc108716f705, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f] 2024-11-10T09:17:52,609 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3384916ea534f3398dfcc108716f705, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731230267752 2024-11-10T09:17:52,609 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471e0dde4bb54619bba562fd7b89de20, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731230268958 2024-11-10T09:17:52,610 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb9e49de1cec4bb688fd90ea02c5311f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230269763 2024-11-10T09:17:52,624 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:52,624 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a81cb04023d94387938da955cafccd3d is 50, key is test_row_0/B:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:52,635 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:52,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742214_1390 (size=12493) 2024-11-10T09:17:52,638 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100b2a50b0c3ff4674b6f85a37fda3568b_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:52,640 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100b2a50b0c3ff4674b6f85a37fda3568b_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:52,640 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100b2a50b0c3ff4674b6f85a37fda3568b_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:52,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742215_1391 (size=4469) 2024-11-10T09:17:52,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T09:17:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,713 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T09:17:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:52,748 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#331 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:52,749 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/83c83589ff4446daaa228be9a7baa0c0 is 175, key is test_row_0/A:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:52,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103e6f27aef5f44fa8ad33303fe7d0d638_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230271053/Put/seqid=0 2024-11-10T09:17:52,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742216_1392 (size=31447) 2024-11-10T09:17:52,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742217_1393 (size=12304) 2024-11-10T09:17:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:52,833 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/83c83589ff4446daaa228be9a7baa0c0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/83c83589ff4446daaa228be9a7baa0c0 2024-11-10T09:17:52,841 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103e6f27aef5f44fa8ad33303fe7d0d638_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e6f27aef5f44fa8ad33303fe7d0d638_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:52,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0a6103f284984df9b5951ddf73f3530a, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:52,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0a6103f284984df9b5951ddf73f3530a is 175, key is test_row_0/A:col10/1731230271053/Put/seqid=0 2024-11-10T09:17:52,880 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into 83c83589ff4446daaa228be9a7baa0c0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:52,880 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:52,880 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230272605; duration=0sec 2024-11-10T09:17:52,880 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:52,880 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:17:52,880 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:52,887 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:52,887 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:17:52,887 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:52,887 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/2f4adb079ae142c699a969ad68d160d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/a7cefb22863441909360af0029e8066e] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=35.8 K 2024-11-10T09:17:52,888 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4adb079ae142c699a969ad68d160d7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731230267752 2024-11-10T09:17:52,892 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1a8b0d5a98e46aabd9a2e7d3208f1e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731230268958 2024-11-10T09:17:52,893 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7cefb22863441909360af0029e8066e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230269779 2024-11-10T09:17:52,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742218_1394 (size=31105) 2024-11-10T09:17:52,900 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0a6103f284984df9b5951ddf73f3530a 2024-11-10T09:17:52,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/29e460fc152c4437943b6cd3e4ba52ed is 50, key is test_row_0/B:col10/1731230271053/Put/seqid=0 2024-11-10T09:17:52,918 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:52,919 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/f59250fe5ec445f88d5b7018245045ec is 50, key is test_row_0/C:col10/1731230271022/Put/seqid=0 2024-11-10T09:17:52,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742219_1395 (size=12151) 2024-11-10T09:17:52,933 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/29e460fc152c4437943b6cd3e4ba52ed 2024-11-10T09:17:52,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742220_1396 (size=12493) 2024-11-10T09:17:52,986 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/f59250fe5ec445f88d5b7018245045ec as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f59250fe5ec445f88d5b7018245045ec 2024-11-10T09:17:52,991 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into f59250fe5ec445f88d5b7018245045ec(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:52,991 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:52,992 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230272605; duration=0sec 2024-11-10T09:17:52,992 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:52,992 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:17:53,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/8b307af7ed2e459d91a4040a06b4a9e4 is 50, key is test_row_0/C:col10/1731230271053/Put/seqid=0 2024-11-10T09:17:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742221_1397 (size=12151) 2024-11-10T09:17:53,029 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/8b307af7ed2e459d91a4040a06b4a9e4 2024-11-10T09:17:53,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0a6103f284984df9b5951ddf73f3530a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a 2024-11-10T09:17:53,044 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a, entries=150, sequenceid=204, filesize=30.4 K 2024-11-10T09:17:53,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/29e460fc152c4437943b6cd3e4ba52ed as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/29e460fc152c4437943b6cd3e4ba52ed 2024-11-10T09:17:53,048 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/a81cb04023d94387938da955cafccd3d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a81cb04023d94387938da955cafccd3d 2024-11-10T09:17:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,054 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into a81cb04023d94387938da955cafccd3d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:53,054 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:53,054 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230272605; duration=0sec 2024-11-10T09:17:53,054 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:53,054 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:17:53,057 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/29e460fc152c4437943b6cd3e4ba52ed, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T09:17:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/8b307af7ed2e459d91a4040a06b4a9e4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/8b307af7ed2e459d91a4040a06b4a9e4 2024-11-10T09:17:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,079 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/8b307af7ed2e459d91a4040a06b4a9e4, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T09:17:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,081 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 367ms, sequenceid=204, compaction requested=false 2024-11-10T09:17:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T09:17:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T09:17:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T09:17:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1110 sec 2024-11-10T09:17:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.1180 sec 2024-11-10T09:17:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:53,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:17:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:53,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:53,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:53,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:53,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103ef61b769880450ab56aaf6f70b808df_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742222_1398 (size=19774) 2024-11-10T09:17:53,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:53,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103ef61b769880450ab56aaf6f70b808df_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103ef61b769880450ab56aaf6f70b808df_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:53,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/fdfe93f22698486c89f249c19a8797e0, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:53,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/fdfe93f22698486c89f249c19a8797e0 is 175, key is test_row_0/A:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742223_1399 (size=57033) 2024-11-10T09:17:53,613 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/fdfe93f22698486c89f249c19a8797e0 2024-11-10T09:17:53,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230273585, max timestamp=1731230273621Num uniques 859; 2024-11-10T09:17:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230273585, max timestamp=1731230273621Num uniques 859; 2024-11-10T09:17:53,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:53,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230273585, max timestamp=1731230273621Num uniques 859; 2024-11-10T09:17:53,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:17:53,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/8c8ad4a262024a0dbdcbfbe938fe4ca3 is 50, key is test_row_0/B:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:53,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742224_1400 (size=12151) 2024-11-10T09:17:53,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/8c8ad4a262024a0dbdcbfbe938fe4ca3 2024-11-10T09:17:53,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230333668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230333677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230333675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230333682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230333698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/7bd18704cc9141a19f1363ac4bac4e07 is 50, key is test_row_0/C:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:53,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742225_1401 (size=12151) 2024-11-10T09:17:53,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230333813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230333813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230333813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230333821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230333832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230334058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230334058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230334058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230334058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230334059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/7bd18704cc9141a19f1363ac4bac4e07 2024-11-10T09:17:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T09:17:54,161 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:54,161 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-10T09:17:54,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:54,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:54,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T09:17:54,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/fdfe93f22698486c89f249c19a8797e0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0 2024-11-10T09:17:54,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0, entries=300, sequenceid=218, filesize=55.7 K 2024-11-10T09:17:54,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/8c8ad4a262024a0dbdcbfbe938fe4ca3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8c8ad4a262024a0dbdcbfbe938fe4ca3 2024-11-10T09:17:54,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8c8ad4a262024a0dbdcbfbe938fe4ca3, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T09:17:54,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/7bd18704cc9141a19f1363ac4bac4e07 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/7bd18704cc9141a19f1363ac4bac4e07 2024-11-10T09:17:54,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/7bd18704cc9141a19f1363ac4bac4e07, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T09:17:54,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 782ms, sequenceid=218, compaction requested=true 2024-11-10T09:17:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:54,274 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:17:54,275 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:54,275 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:17:54,275 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:54,275 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:54,276 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/83c83589ff4446daaa228be9a7baa0c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=116.8 K 2024-11-10T09:17:54,276 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:54,276 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/83c83589ff4446daaa228be9a7baa0c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0] 2024-11-10T09:17:54,276 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 83c83589ff4446daaa228be9a7baa0c0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230269779 2024-11-10T09:17:54,277 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6103f284984df9b5951ddf73f3530a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731230271033 2024-11-10T09:17:54,277 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting fdfe93f22698486c89f249c19a8797e0, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731230273349 2024-11-10T09:17:54,277 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:54,277 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:17:54,277 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:54,277 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a81cb04023d94387938da955cafccd3d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/29e460fc152c4437943b6cd3e4ba52ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8c8ad4a262024a0dbdcbfbe938fe4ca3] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=35.9 K 2024-11-10T09:17:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T09:17:54,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81cb04023d94387938da955cafccd3d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230269779 2024-11-10T09:17:54,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e460fc152c4437943b6cd3e4ba52ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731230271033 2024-11-10T09:17:54,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8ad4a262024a0dbdcbfbe938fe4ca3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731230273354 2024-11-10T09:17:54,292 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:54,298 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:54,300 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/54e41d90322e4d0598b95a2ff9696e3a is 50, key is test_row_0/B:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:54,304 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ea408ff58de2475492faa7015985f358_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:54,306 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ea408ff58de2475492faa7015985f358_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:54,306 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ea408ff58de2475492faa7015985f358_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:54,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T09:17:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:54,324 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-10T09:17:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:54,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742226_1402 (size=12595) 2024-11-10T09:17:54,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742227_1403 (size=4469) 2024-11-10T09:17:54,377 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#339 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076ecaf66285146dca88fb702421bd3ac_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230273687/Put/seqid=0 2024-11-10T09:17:54,382 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/8d17eabb43da4f00bbca827cba84d0e0 is 175, key is test_row_0/A:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:54,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:54,400 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/54e41d90322e4d0598b95a2ff9696e3a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/54e41d90322e4d0598b95a2ff9696e3a 2024-11-10T09:17:54,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230334400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230334401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230334404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,433 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into 54e41d90322e4d0598b95a2ff9696e3a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:54,433 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:54,433 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230274275; duration=0sec 2024-11-10T09:17:54,433 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:54,433 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:17:54,433 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:54,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230334415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,443 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:54,443 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:17:54,443 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:54,443 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f59250fe5ec445f88d5b7018245045ec, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/8b307af7ed2e459d91a4040a06b4a9e4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/7bd18704cc9141a19f1363ac4bac4e07] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=35.9 K 2024-11-10T09:17:54,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230334425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,448 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59250fe5ec445f88d5b7018245045ec, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230269779 2024-11-10T09:17:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,452 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b307af7ed2e459d91a4040a06b4a9e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731230271033 2024-11-10T09:17:54,453 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bd18704cc9141a19f1363ac4bac4e07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731230273354 2024-11-10T09:17:54,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742229_1405 (size=31549) 2024-11-10T09:17:54,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742228_1404 (size=12304) 2024-11-10T09:17:54,470 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:54,471 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/dab89e04f8334f6d8f14cbd207e6b77f is 50, key is test_row_0/C:col10/1731230273478/Put/seqid=0 2024-11-10T09:17:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T09:17:54,488 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/8d17eabb43da4f00bbca827cba84d0e0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d17eabb43da4f00bbca827cba84d0e0 2024-11-10T09:17:54,501 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into 8d17eabb43da4f00bbca827cba84d0e0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:54,501 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:54,501 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230274274; duration=0sec 2024-11-10T09:17:54,502 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:54,502 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:17:54,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742230_1406 (size=12595) 2024-11-10T09:17:54,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230334527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230334528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230334528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230334549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230334557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230334747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230334747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230334747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230334767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230334778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T09:17:54,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:54,875 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076ecaf66285146dca88fb702421bd3ac_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076ecaf66285146dca88fb702421bd3ac_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/8d745a2f81e047ffa9a59bd37169c38c, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:54,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/8d745a2f81e047ffa9a59bd37169c38c is 175, key is test_row_0/A:col10/1731230273687/Put/seqid=0 2024-11-10T09:17:54,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742231_1407 (size=31105) 2024-11-10T09:17:54,923 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/dab89e04f8334f6d8f14cbd207e6b77f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/dab89e04f8334f6d8f14cbd207e6b77f 2024-11-10T09:17:54,933 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into dab89e04f8334f6d8f14cbd207e6b77f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:54,933 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:54,933 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230274275; duration=0sec 2024-11-10T09:17:54,933 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:54,933 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:17:55,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230335067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230335068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230335068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230335078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230335097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,285 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/8d745a2f81e047ffa9a59bd37169c38c 2024-11-10T09:17:55,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/db3f6c7598994da880d036accb7dd92b is 50, key is test_row_0/B:col10/1731230273687/Put/seqid=0 2024-11-10T09:17:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T09:17:55,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742232_1408 (size=12151) 2024-11-10T09:17:55,356 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/db3f6c7598994da880d036accb7dd92b 2024-11-10T09:17:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/0bf00d15eef144bbade3c468d4c66e1a is 50, key is test_row_0/C:col10/1731230273687/Put/seqid=0 2024-11-10T09:17:55,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742233_1409 (size=12151) 2024-11-10T09:17:55,389 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/0bf00d15eef144bbade3c468d4c66e1a 2024-11-10T09:17:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/8d745a2f81e047ffa9a59bd37169c38c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c 2024-11-10T09:17:55,414 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c, entries=150, sequenceid=252, filesize=30.4 K 2024-11-10T09:17:55,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/db3f6c7598994da880d036accb7dd92b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/db3f6c7598994da880d036accb7dd92b 2024-11-10T09:17:55,442 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/db3f6c7598994da880d036accb7dd92b, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T09:17:55,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/0bf00d15eef144bbade3c468d4c66e1a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0bf00d15eef144bbade3c468d4c66e1a 2024-11-10T09:17:55,456 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0bf00d15eef144bbade3c468d4c66e1a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T09:17:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1133ms, sequenceid=252, compaction requested=false 2024-11-10T09:17:55,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:55,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:55,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-10T09:17:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-10T09:17:55,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T09:17:55,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3060 sec 2024-11-10T09:17:55,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.3140 sec 2024-11-10T09:17:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:55,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:17:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:55,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051ce7aaac9f440598bc6b7fbfd31f6ed_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:55,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742234_1410 (size=14994) 2024-11-10T09:17:55,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230335623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230335626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230275620, max timestamp=1731230275637Num uniques 859; 2024-11-10T09:17:55,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230275620, max timestamp=1731230275637Num uniques 859; 2024-11-10T09:17:55,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:55,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:55,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230275620, max timestamp=1731230275637Num uniques 859; 2024-11-10T09:17:55,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230335719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230335719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230335721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230335747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230335748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230335848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230335848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230335848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230335967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:55,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230335971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:56,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051ce7aaac9f440598bc6b7fbfd31f6ed_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051ce7aaac9f440598bc6b7fbfd31f6ed_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:56,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/138d84ca3da942b7be6cf0da967e8ec8, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:56,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/138d84ca3da942b7be6cf0da967e8ec8 is 175, key is test_row_0/A:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:56,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742235_1411 (size=39949) 2024-11-10T09:17:56,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/138d84ca3da942b7be6cf0da967e8ec8 2024-11-10T09:17:56,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/d67555b9de4945b7b6838d68cdf0b46f is 50, key is test_row_0/B:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:56,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742236_1412 (size=12301) 2024-11-10T09:17:56,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230336067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230336067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230336068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230336288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230336300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T09:17:56,318 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:56,318 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-10T09:17:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T09:17:56,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:56,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:56,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230336398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230336398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230336399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T09:17:56,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/d67555b9de4945b7b6838d68cdf0b46f 2024-11-10T09:17:56,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/5d7557a91bd3409fa9a8dc181eef793b is 50, key is test_row_0/C:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:56,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T09:17:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742237_1413 (size=12301) 2024-11-10T09:17:56,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T09:17:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T09:17:56,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T09:17:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:56,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230336797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230336817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/5d7557a91bd3409fa9a8dc181eef793b 2024-11-10T09:17:56,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/138d84ca3da942b7be6cf0da967e8ec8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8 2024-11-10T09:17:56,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8, entries=200, sequenceid=267, filesize=39.0 K 2024-11-10T09:17:56,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/d67555b9de4945b7b6838d68cdf0b46f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/d67555b9de4945b7b6838d68cdf0b46f 2024-11-10T09:17:56,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/d67555b9de4945b7b6838d68cdf0b46f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T09:17:56,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/5d7557a91bd3409fa9a8dc181eef793b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/5d7557a91bd3409fa9a8dc181eef793b 2024-11-10T09:17:56,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230336918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:56,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/5d7557a91bd3409fa9a8dc181eef793b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T09:17:56,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1348ms, sequenceid=267, compaction requested=true 2024-11-10T09:17:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:56,933 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:56,934 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:56,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:56,934 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:56,934 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:17:56,934 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,934 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d17eabb43da4f00bbca827cba84d0e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=100.2 K 2024-11-10T09:17:56,935 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,935 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d17eabb43da4f00bbca827cba84d0e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8] 2024-11-10T09:17:56,935 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:56,935 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:17:56,935 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,935 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/54e41d90322e4d0598b95a2ff9696e3a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/db3f6c7598994da880d036accb7dd92b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/d67555b9de4945b7b6838d68cdf0b46f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.2 K 2024-11-10T09:17:56,936 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d17eabb43da4f00bbca827cba84d0e0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731230273354 2024-11-10T09:17:56,936 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d745a2f81e047ffa9a59bd37169c38c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230273666 2024-11-10T09:17:56,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T09:17:56,936 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e41d90322e4d0598b95a2ff9696e3a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731230273354 2024-11-10T09:17:56,937 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting db3f6c7598994da880d036accb7dd92b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230273666 2024-11-10T09:17:56,937 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 138d84ca3da942b7be6cf0da967e8ec8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731230274397 2024-11-10T09:17:56,937 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d67555b9de4945b7b6838d68cdf0b46f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731230274397 2024-11-10T09:17:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:56,940 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-10T09:17:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:17:56,941 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:56,943 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:56,943 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/ffef22d7380a46289854e6fcfea10cd9 is 50, key is test_row_0/B:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:56,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:56,947 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411104366a2add7ce4ddea2ed19aae93706ac_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T09:17:56,949 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411104366a2add7ce4ddea2ed19aae93706ac_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:56,949 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104366a2add7ce4ddea2ed19aae93706ac_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:56,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d041243601684ceeaf70a98acd92c18f_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230275692/Put/seqid=0 2024-11-10T09:17:56,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742239_1415 (size=4469) 2024-11-10T09:17:56,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742238_1414 (size=12847) 2024-11-10T09:17:56,960 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#349 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:56,960 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/3409a074f7a14fde8fed34d217380eaf is 175, key is test_row_0/A:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:56,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742240_1416 (size=12454) 2024-11-10T09:17:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:56,972 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d041243601684ceeaf70a98acd92c18f_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d041243601684ceeaf70a98acd92c18f_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:56,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/db0a47e3db394f4082cf9d265e9bc953, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/db0a47e3db394f4082cf9d265e9bc953 is 175, key is test_row_0/A:col10/1731230275692/Put/seqid=0 2024-11-10T09:17:56,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742241_1417 (size=31801) 2024-11-10T09:17:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742242_1418 (size=31255) 2024-11-10T09:17:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230337035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230337039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230337147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230337157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,364 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/ffef22d7380a46289854e6fcfea10cd9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ffef22d7380a46289854e6fcfea10cd9 2024-11-10T09:17:57,374 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into ffef22d7380a46289854e6fcfea10cd9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:57,374 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:57,374 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230276933; duration=0sec 2024-11-10T09:17:57,374 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:57,374 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:17:57,374 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:57,375 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:57,375 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:17:57,375 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:57,375 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/dab89e04f8334f6d8f14cbd207e6b77f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0bf00d15eef144bbade3c468d4c66e1a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/5d7557a91bd3409fa9a8dc181eef793b] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.2 K 2024-11-10T09:17:57,375 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting dab89e04f8334f6d8f14cbd207e6b77f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731230273354 2024-11-10T09:17:57,375 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf00d15eef144bbade3c468d4c66e1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230273666 2024-11-10T09:17:57,376 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7557a91bd3409fa9a8dc181eef793b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731230274397 2024-11-10T09:17:57,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230337367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,384 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/db0a47e3db394f4082cf9d265e9bc953 2024-11-10T09:17:57,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230337377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,394 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/3409a074f7a14fde8fed34d217380eaf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3409a074f7a14fde8fed34d217380eaf 2024-11-10T09:17:57,400 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into 3409a074f7a14fde8fed34d217380eaf(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:57,400 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:57,400 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230276933; duration=0sec 2024-11-10T09:17:57,400 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:57,400 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:17:57,409 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#351 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:57,410 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/cfe89e79bd514770be083958c898fd06 is 50, key is test_row_0/C:col10/1731230274420/Put/seqid=0 2024-11-10T09:17:57,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/78d863fe63214b05afff6b5afb3473b1 is 50, key is test_row_0/B:col10/1731230275692/Put/seqid=0 2024-11-10T09:17:57,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742244_1420 (size=12847) 2024-11-10T09:17:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742243_1419 (size=12301) 2024-11-10T09:17:57,444 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/78d863fe63214b05afff6b5afb3473b1 2024-11-10T09:17:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/725ab006c6bb4396bac0aed496ddc338 is 50, key is test_row_0/C:col10/1731230275692/Put/seqid=0 2024-11-10T09:17:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T09:17:57,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742245_1421 (size=12301) 2024-11-10T09:17:57,473 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/725ab006c6bb4396bac0aed496ddc338 2024-11-10T09:17:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/db0a47e3db394f4082cf9d265e9bc953 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953 2024-11-10T09:17:57,484 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953, entries=150, sequenceid=301, filesize=30.5 K 2024-11-10T09:17:57,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/78d863fe63214b05afff6b5afb3473b1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/78d863fe63214b05afff6b5afb3473b1 2024-11-10T09:17:57,489 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/78d863fe63214b05afff6b5afb3473b1, entries=150, sequenceid=301, filesize=12.0 K 2024-11-10T09:17:57,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/725ab006c6bb4396bac0aed496ddc338 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/725ab006c6bb4396bac0aed496ddc338 2024-11-10T09:17:57,504 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/725ab006c6bb4396bac0aed496ddc338, entries=150, sequenceid=301, filesize=12.0 K 2024-11-10T09:17:57,505 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 565ms, sequenceid=301, compaction requested=false 2024-11-10T09:17:57,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:57,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:57,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-10T09:17:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-10T09:17:57,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T09:17:57,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1850 sec 2024-11-10T09:17:57,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1910 sec 2024-11-10T09:17:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:57,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:17:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:57,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8d703e3e1cc44228b84f3a5464d25ac_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:57,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742246_1422 (size=14994) 2024-11-10T09:17:57,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:57,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8d703e3e1cc44228b84f3a5464d25ac_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8d703e3e1cc44228b84f3a5464d25ac_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:57,809 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0ca55e58014e4a0a9d72a1aac775ae0c, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:57,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0ca55e58014e4a0a9d72a1aac775ae0c is 175, key is test_row_0/A:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:57,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742247_1423 (size=39949) 2024-11-10T09:17:57,848 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/cfe89e79bd514770be083958c898fd06 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cfe89e79bd514770be083958c898fd06 2024-11-10T09:17:57,854 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into cfe89e79bd514770be083958c898fd06(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:57,854 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:57,854 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230276934; duration=0sec 2024-11-10T09:17:57,855 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:57,855 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:17:57,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230337838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230337838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,857 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 32f231a2c762c9b2ac161a81d8fdd9d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230337842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230337850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230337947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,951 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 32f231a2c762c9b2ac161a81d8fdd9d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230337968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230337972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230337977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230338187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230338187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230338187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0ca55e58014e4a0a9d72a1aac775ae0c 2024-11-10T09:17:58,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/62d50ae902424abd8da7073e137e1a38 is 50, key is test_row_0/B:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742248_1424 (size=12301) 2024-11-10T09:17:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T09:17:58,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:17:58,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:17:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T09:17:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:17:58,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:17:58,470 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:17:58,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:17:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230338508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230338508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:58,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230338508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:17:58,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:17:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:58,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:17:58,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/62d50ae902424abd8da7073e137e1a38 2024-11-10T09:17:58,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/f37b082423e24172987e381ca55188b5 is 50, key is test_row_0/C:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:58,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742249_1425 (size=12301) 2024-11-10T09:17:58,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/f37b082423e24172987e381ca55188b5 2024-11-10T09:17:58,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/0ca55e58014e4a0a9d72a1aac775ae0c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c 2024-11-10T09:17:58,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c, entries=200, sequenceid=316, filesize=39.0 K 2024-11-10T09:17:58,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/62d50ae902424abd8da7073e137e1a38 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/62d50ae902424abd8da7073e137e1a38 2024-11-10T09:17:58,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/62d50ae902424abd8da7073e137e1a38, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T09:17:58,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/f37b082423e24172987e381ca55188b5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f37b082423e24172987e381ca55188b5 2024-11-10T09:17:58,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f37b082423e24172987e381ca55188b5, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T09:17:58,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1043ms, sequenceid=316, compaction requested=true 2024-11-10T09:17:58,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:58,736 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:58,737 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:58,737 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:17:58,737 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:58,737 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3409a074f7a14fde8fed34d217380eaf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=100.6 K 2024-11-10T09:17:58,737 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:58,738 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3409a074f7a14fde8fed34d217380eaf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c] 2024-11-10T09:17:58,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:17:58,738 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3409a074f7a14fde8fed34d217380eaf, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731230274397 2024-11-10T09:17:58,738 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting db0a47e3db394f4082cf9d265e9bc953, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731230275692 2024-11-10T09:17:58,739 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca55e58014e4a0a9d72a1aac775ae0c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731230277032 2024-11-10T09:17:58,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:58,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:17:58,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:58,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:17:58,739 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:58,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:58,764 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:58,764 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:17:58,764 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:58,764 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ffef22d7380a46289854e6fcfea10cd9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/78d863fe63214b05afff6b5afb3473b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/62d50ae902424abd8da7073e137e1a38] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.6 K 2024-11-10T09:17:58,765 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ffef22d7380a46289854e6fcfea10cd9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731230274397 2024-11-10T09:17:58,766 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d863fe63214b05afff6b5afb3473b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731230275692 2024-11-10T09:17:58,766 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d50ae902424abd8da7073e137e1a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731230277032 2024-11-10T09:17:58,773 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:58,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:17:58,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:58,777 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T09:17:58,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:17:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:17:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:17:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:17:58,784 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:58,785 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/374b045e25684b55ab29db8765e2b0ed is 50, key is test_row_0/B:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:58,787 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102634b33aba0e4d2d9ba8bff9b940e533_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:17:58,789 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102634b33aba0e4d2d9ba8bff9b940e533_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:58,789 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102634b33aba0e4d2d9ba8bff9b940e533_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:58,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742251_1427 (size=4469) 2024-11-10T09:17:58,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b6f7946f085744b9963c79217e08d46a_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230277785/Put/seqid=0 2024-11-10T09:17:58,816 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#357 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:58,816 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/faca53c3d48d45aba5cd6f698aadc057 is 175, key is test_row_0/A:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:58,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742250_1426 (size=12949) 2024-11-10T09:17:58,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742252_1428 (size=12454) 2024-11-10T09:17:58,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742253_1429 (size=31903) 2024-11-10T09:17:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:59,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:17:59,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230339072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230339073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230339082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:17:59,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230339189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230339189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230339199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,230 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/374b045e25684b55ab29db8765e2b0ed as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/374b045e25684b55ab29db8765e2b0ed 2024-11-10T09:17:59,237 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into 374b045e25684b55ab29db8765e2b0ed(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:59,237 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:59,237 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230278739; duration=0sec 2024-11-10T09:17:59,237 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:17:59,237 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:17:59,237 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:17:59,239 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:17:59,239 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:17:59,239 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:17:59,239 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cfe89e79bd514770be083958c898fd06, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/725ab006c6bb4396bac0aed496ddc338, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f37b082423e24172987e381ca55188b5] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.6 K 2024-11-10T09:17:59,240 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe89e79bd514770be083958c898fd06, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731230274397 2024-11-10T09:17:59,240 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 725ab006c6bb4396bac0aed496ddc338, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731230275692 2024-11-10T09:17:59,240 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f37b082423e24172987e381ca55188b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731230277032 2024-11-10T09:17:59,250 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:17:59,250 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/c667cad5a8a64c718db037dfa26c6e9e is 50, key is test_row_0/C:col10/1731230277691/Put/seqid=0 2024-11-10T09:17:59,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:17:59,262 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b6f7946f085744b9963c79217e08d46a_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b6f7946f085744b9963c79217e08d46a_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:17:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/979f33f486da4c9da2517af8270953ee, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:17:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/979f33f486da4c9da2517af8270953ee is 175, key is test_row_0/A:col10/1731230277785/Put/seqid=0 2024-11-10T09:17:59,271 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/faca53c3d48d45aba5cd6f698aadc057 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/faca53c3d48d45aba5cd6f698aadc057 2024-11-10T09:17:59,279 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into faca53c3d48d45aba5cd6f698aadc057(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:59,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:59,279 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230278736; duration=0sec 2024-11-10T09:17:59,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:59,279 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:17:59,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742255_1431 (size=31255) 2024-11-10T09:17:59,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742254_1430 (size=12949) 2024-11-10T09:17:59,325 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/c667cad5a8a64c718db037dfa26c6e9e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/c667cad5a8a64c718db037dfa26c6e9e 2024-11-10T09:17:59,343 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into c667cad5a8a64c718db037dfa26c6e9e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:17:59,343 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:17:59,343 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230278739; duration=0sec 2024-11-10T09:17:59,343 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:17:59,343 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:17:59,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230339408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230339408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230339417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:17:59,700 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/979f33f486da4c9da2517af8270953ee 2024-11-10T09:17:59,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5474217591da44a99db423f20ce4dcfb is 50, key is test_row_0/B:col10/1731230277785/Put/seqid=0 2024-11-10T09:17:59,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230339728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230339737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742256_1432 (size=12301) 2024-11-10T09:17:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230339747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57108 deadline: 1731230339877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,890 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 32f231a2c762c9b2ac161a81d8fdd9d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:17:59,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:17:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57112 deadline: 1731230339968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:17:59,980 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 32f231a2c762c9b2ac161a81d8fdd9d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:17:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:17:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,151 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5474217591da44a99db423f20ce4dcfb 2024-11-10T09:18:00,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/fc56aae91ccf447dbd5987bc42f32e98 is 50, key is test_row_0/C:col10/1731230277785/Put/seqid=0 2024-11-10T09:18:00,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742257_1433 (size=12301) 2024-11-10T09:18:00,182 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/fc56aae91ccf447dbd5987bc42f32e98 2024-11-10T09:18:00,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/979f33f486da4c9da2517af8270953ee as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee 2024-11-10T09:18:00,203 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee, entries=150, sequenceid=336, filesize=30.5 K 2024-11-10T09:18:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5474217591da44a99db423f20ce4dcfb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5474217591da44a99db423f20ce4dcfb 2024-11-10T09:18:00,211 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5474217591da44a99db423f20ce4dcfb, entries=150, sequenceid=336, filesize=12.0 K 2024-11-10T09:18:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/fc56aae91ccf447dbd5987bc42f32e98 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/fc56aae91ccf447dbd5987bc42f32e98 2024-11-10T09:18:00,218 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/fc56aae91ccf447dbd5987bc42f32e98, entries=150, sequenceid=336, filesize=12.0 K 2024-11-10T09:18:00,221 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1444ms, sequenceid=336, compaction requested=false 2024-11-10T09:18:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T09:18:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T09:18:00,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T09:18:00,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-10T09:18:00,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7590 sec 2024-11-10T09:18:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:00,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:18:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:18:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:18:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:18:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:00,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c35f91d7cf34c89b5fb73c14114a6d6_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:00,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742258_1434 (size=14994) 2024-11-10T09:18:00,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:00,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c35f91d7cf34c89b5fb73c14114a6d6_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c35f91d7cf34c89b5fb73c14114a6d6_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:00,292 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/11c8ca086ad44edeba0fe1f4ec88a675, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:00,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/11c8ca086ad44edeba0fe1f4ec88a675 is 175, key is test_row_0/A:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742259_1435 (size=39949) 2024-11-10T09:18:00,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230340306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230340307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230340313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230340428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230340428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230340428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:18:00,618 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:00,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T09:18:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:18:00,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:00,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:00,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230340649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230340649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230340649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:00,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/11c8ca086ad44edeba0fe1f4ec88a675 2024-11-10T09:18:00,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/eee0363c1b9247d3bcd00173bc41ef69 is 50, key is test_row_0/B:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:18:00,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742260_1436 (size=12301) 2024-11-10T09:18:00,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/eee0363c1b9247d3bcd00173bc41ef69 2024-11-10T09:18:00,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T09:18:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:18:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:00,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/4f224be6938d4c94b55212befa9b915f is 50, key is test_row_0/C:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:00,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742261_1437 (size=12301) 2024-11-10T09:18:00,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/4f224be6938d4c94b55212befa9b915f 2024-11-10T09:18:00,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/11c8ca086ad44edeba0fe1f4ec88a675 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675 2024-11-10T09:18:00,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675, entries=200, sequenceid=353, filesize=39.0 K 2024-11-10T09:18:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/eee0363c1b9247d3bcd00173bc41ef69 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/eee0363c1b9247d3bcd00173bc41ef69 2024-11-10T09:18:00,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/eee0363c1b9247d3bcd00173bc41ef69, entries=150, sequenceid=353, filesize=12.0 K 2024-11-10T09:18:00,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/4f224be6938d4c94b55212befa9b915f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4f224be6938d4c94b55212befa9b915f 2024-11-10T09:18:00,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4f224be6938d4c94b55212befa9b915f, entries=150, sequenceid=353, filesize=12.0 K 2024-11-10T09:18:00,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 636ms, sequenceid=353, compaction requested=true 2024-11-10T09:18:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:00,885 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:00,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:00,887 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:00,887 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:18:00,887 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,887 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/faca53c3d48d45aba5cd6f698aadc057, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=100.7 K 2024-11-10T09:18:00,887 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,887 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/faca53c3d48d45aba5cd6f698aadc057, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675] 2024-11-10T09:18:00,888 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:00,888 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:18:00,888 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,888 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/374b045e25684b55ab29db8765e2b0ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5474217591da44a99db423f20ce4dcfb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/eee0363c1b9247d3bcd00173bc41ef69] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.7 K 2024-11-10T09:18:00,888 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting faca53c3d48d45aba5cd6f698aadc057, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731230277032 2024-11-10T09:18:00,888 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 374b045e25684b55ab29db8765e2b0ed, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731230277032 2024-11-10T09:18:00,888 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979f33f486da4c9da2517af8270953ee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731230277785 2024-11-10T09:18:00,889 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5474217591da44a99db423f20ce4dcfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731230277785 2024-11-10T09:18:00,889 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c8ca086ad44edeba0fe1f4ec88a675, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731230279070 2024-11-10T09:18:00,889 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting eee0363c1b9247d3bcd00173bc41ef69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731230279070 2024-11-10T09:18:00,917 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:00,932 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:00,933 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/15060185904e498eafd375f89559df06 is 50, key is test_row_0/B:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:18:00,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T09:18:00,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:00,940 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T09:18:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:18:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:18:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:18:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:00,953 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111057de129d02be48c0a211fa129476ef87_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:00,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111057de129d02be48c0a211fa129476ef87_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:00,955 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111057de129d02be48c0a211fa129476ef87_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:00,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:18:00,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742262_1438 (size=13051) 2024-11-10T09:18:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014040862d51d43479ccf1afaa61d8bb9_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230280306/Put/seqid=0 2024-11-10T09:18:01,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742263_1439 (size=4469) 2024-11-10T09:18:01,010 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#366 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:01,010 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/13a0a8687a854225b6239af3c50c6107 is 175, key is test_row_0/A:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742264_1440 (size=12454) 2024-11-10T09:18:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:01,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742265_1441 (size=32005) 2024-11-10T09:18:01,054 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014040862d51d43479ccf1afaa61d8bb9_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014040862d51d43479ccf1afaa61d8bb9_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/f0f4c9cb83af4a20bf5306e09a4369e2, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/f0f4c9cb83af4a20bf5306e09a4369e2 is 175, key is test_row_0/A:col10/1731230280306/Put/seqid=0 2024-11-10T09:18:01,061 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/13a0a8687a854225b6239af3c50c6107 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/13a0a8687a854225b6239af3c50c6107 2024-11-10T09:18:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230341054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,066 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into 13a0a8687a854225b6239af3c50c6107(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:01,066 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:01,066 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230280885; duration=0sec 2024-11-10T09:18:01,066 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:01,066 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:18:01,066 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:01,067 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:01,067 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:18:01,067 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:01,067 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/c667cad5a8a64c718db037dfa26c6e9e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/fc56aae91ccf447dbd5987bc42f32e98, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4f224be6938d4c94b55212befa9b915f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.7 K 2024-11-10T09:18:01,068 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c667cad5a8a64c718db037dfa26c6e9e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731230277032 2024-11-10T09:18:01,068 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc56aae91ccf447dbd5987bc42f32e98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731230277785 2024-11-10T09:18:01,068 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f224be6938d4c94b55212befa9b915f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731230279070 2024-11-10T09:18:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230341056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742266_1442 (size=31255) 2024-11-10T09:18:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230341062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,086 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:01,086 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/12b14871beff4820a81ca19c73dc16d4 is 50, key is test_row_0/C:col10/1731230279080/Put/seqid=0 2024-11-10T09:18:01,087 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/f0f4c9cb83af4a20bf5306e09a4369e2 2024-11-10T09:18:01,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742267_1443 (size=13051) 2024-11-10T09:18:01,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/8790f0dd81ee40d688b9535303e2a35e is 50, key is test_row_0/B:col10/1731230280306/Put/seqid=0 2024-11-10T09:18:01,156 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/12b14871beff4820a81ca19c73dc16d4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/12b14871beff4820a81ca19c73dc16d4 2024-11-10T09:18:01,169 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into 12b14871beff4820a81ca19c73dc16d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:01,169 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:01,169 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230280886; duration=0sec 2024-11-10T09:18:01,169 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:01,169 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:18:01,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742268_1444 (size=12301) 2024-11-10T09:18:01,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230341168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,188 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/8790f0dd81ee40d688b9535303e2a35e 2024-11-10T09:18:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230341187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230341187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/356d95dbc65a43b9a69b6dc3dad048a5 is 50, key is test_row_0/C:col10/1731230280306/Put/seqid=0 2024-11-10T09:18:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:18:01,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742269_1445 (size=12301) 2024-11-10T09:18:01,277 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/356d95dbc65a43b9a69b6dc3dad048a5 2024-11-10T09:18:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/f0f4c9cb83af4a20bf5306e09a4369e2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2 2024-11-10T09:18:01,288 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2, entries=150, sequenceid=371, filesize=30.5 K 2024-11-10T09:18:01,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/8790f0dd81ee40d688b9535303e2a35e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8790f0dd81ee40d688b9535303e2a35e 2024-11-10T09:18:01,294 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8790f0dd81ee40d688b9535303e2a35e, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:18:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/356d95dbc65a43b9a69b6dc3dad048a5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/356d95dbc65a43b9a69b6dc3dad048a5 2024-11-10T09:18:01,301 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/356d95dbc65a43b9a69b6dc3dad048a5, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:18:01,305 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 365ms, sequenceid=371, compaction requested=false 2024-11-10T09:18:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T09:18:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T09:18:01,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T09:18:01,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 685 msec 2024-11-10T09:18:01,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 691 msec 2024-11-10T09:18:01,400 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/15060185904e498eafd375f89559df06 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/15060185904e498eafd375f89559df06 2024-11-10T09:18:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:01,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T09:18:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:18:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:18:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:18:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:01,405 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into 15060185904e498eafd375f89559df06(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:01,405 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:01,405 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230280885; duration=0sec 2024-11-10T09:18:01,409 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:01,409 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:18:01,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b3a669b039ea48319101cec3fa6f7abf_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742270_1446 (size=14994) 2024-11-10T09:18:01,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230341452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230341460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230341461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230341567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230341568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230341581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:18:01,757 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:01,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T09:18:01,759 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:01,760 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:01,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:18:01,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230341787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230341788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230341798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:01,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:01,834 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b3a669b039ea48319101cec3fa6f7abf_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b3a669b039ea48319101cec3fa6f7abf_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:01,838 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c113dc39655a458193f24d5143561d72, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:01,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c113dc39655a458193f24d5143561d72 is 175, key is test_row_0/A:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742271_1447 (size=39949) 2024-11-10T09:18:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:18:01,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:18:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:18:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:01,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:18:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:18:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:18:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230342107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230342107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230342107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:18:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:18:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:02,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c113dc39655a458193f24d5143561d72 2024-11-10T09:18:02,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5fb5c2c82f854624bab2ae2828d6bd43 is 50, key is test_row_0/B:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:02,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742272_1448 (size=12301) 2024-11-10T09:18:02,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5fb5c2c82f854624bab2ae2828d6bd43 2024-11-10T09:18:02,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/059a914bc8e94ac3b2dbb8165dc7e898 is 50, key is test_row_0/C:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:02,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742273_1449 (size=12301) 2024-11-10T09:18:02,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/059a914bc8e94ac3b2dbb8165dc7e898 2024-11-10T09:18:02,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/c113dc39655a458193f24d5143561d72 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72 2024-11-10T09:18:02,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72, entries=200, sequenceid=388, filesize=39.0 K 2024-11-10T09:18:02,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/5fb5c2c82f854624bab2ae2828d6bd43 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5fb5c2c82f854624bab2ae2828d6bd43 2024-11-10T09:18:02,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5fb5c2c82f854624bab2ae2828d6bd43, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T09:18:02,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/059a914bc8e94ac3b2dbb8165dc7e898 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/059a914bc8e94ac3b2dbb8165dc7e898 2024-11-10T09:18:02,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/059a914bc8e94ac3b2dbb8165dc7e898, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T09:18:02,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 950ms, sequenceid=388, compaction requested=true 2024-11-10T09:18:02,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:02,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:02,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:02,353 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:02,353 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:02,354 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:02,354 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:02,354 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/A is initiating minor compaction (all files) 2024-11-10T09:18:02,354 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/B is initiating minor compaction (all files) 2024-11-10T09:18:02,354 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/A in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,354 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/B in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,354 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/15060185904e498eafd375f89559df06, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8790f0dd81ee40d688b9535303e2a35e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5fb5c2c82f854624bab2ae2828d6bd43] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.8 K 2024-11-10T09:18:02,354 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/13a0a8687a854225b6239af3c50c6107, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=100.8 K 2024-11-10T09:18:02,354 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/13a0a8687a854225b6239af3c50c6107, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72] 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 15060185904e498eafd375f89559df06, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731230279070 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a0a8687a854225b6239af3c50c6107, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731230279070 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8790f0dd81ee40d688b9535303e2a35e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230280305 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f4c9cb83af4a20bf5306e09a4369e2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230280305 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c113dc39655a458193f24d5143561d72, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731230281051 2024-11-10T09:18:02,355 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb5c2c82f854624bab2ae2828d6bd43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731230281054 2024-11-10T09:18:02,363 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:02,365 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:02,365 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/4ef488e18e024fd3b55d72f0fe69a646 is 50, key is test_row_0/B:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:02,369 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c0253faef1444838ad009a7b379b7cc6_32f231a2c762c9b2ac161a81d8fdd9d4 store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:02,371 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c0253faef1444838ad009a7b379b7cc6_32f231a2c762c9b2ac161a81d8fdd9d4, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:02,372 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c0253faef1444838ad009a7b379b7cc6_32f231a2c762c9b2ac161a81d8fdd9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:02,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:18:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,378 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T09:18:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:18:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:18:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:18:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:02,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742275_1451 (size=4469) 2024-11-10T09:18:02,383 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#A#compaction#376 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:02,383 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/631a6b282e7047d48e881d7607dc5f5e is 175, key is test_row_0/A:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:18:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742274_1450 (size=13153) 2024-11-10T09:18:02,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103e48cc08fba4416d8cee6e7bedade6c7_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230281458/Put/seqid=0 2024-11-10T09:18:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742276_1452 (size=32107) 2024-11-10T09:18:02,417 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/631a6b282e7047d48e881d7607dc5f5e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/631a6b282e7047d48e881d7607dc5f5e 2024-11-10T09:18:02,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742277_1453 (size=12454) 2024-11-10T09:18:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:02,433 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/A of 32f231a2c762c9b2ac161a81d8fdd9d4 into 631a6b282e7047d48e881d7607dc5f5e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:02,433 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:02,433 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/A, priority=13, startTime=1731230282353; duration=0sec 2024-11-10T09:18:02,434 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:02,434 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:18:02,434 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:02,435 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:02,435 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 32f231a2c762c9b2ac161a81d8fdd9d4/C is initiating minor compaction (all files) 2024-11-10T09:18:02,435 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 32f231a2c762c9b2ac161a81d8fdd9d4/C in TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:02,435 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/12b14871beff4820a81ca19c73dc16d4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/356d95dbc65a43b9a69b6dc3dad048a5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/059a914bc8e94ac3b2dbb8165dc7e898] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp, totalSize=36.8 K 2024-11-10T09:18:02,437 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103e48cc08fba4416d8cee6e7bedade6c7_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e48cc08fba4416d8cee6e7bedade6c7_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:02,438 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b14871beff4820a81ca19c73dc16d4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731230279070 2024-11-10T09:18:02,444 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356d95dbc65a43b9a69b6dc3dad048a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230280305 2024-11-10T09:18:02,445 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059a914bc8e94ac3b2dbb8165dc7e898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731230281054 2024-11-10T09:18:02,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/298f90a44cc8440ab7082e51ae700a15, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:02,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/298f90a44cc8440ab7082e51ae700a15 is 175, key is test_row_0/A:col10/1731230281458/Put/seqid=0 2024-11-10T09:18:02,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742278_1454 (size=31255) 2024-11-10T09:18:02,487 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/298f90a44cc8440ab7082e51ae700a15 2024-11-10T09:18:02,490 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 32f231a2c762c9b2ac161a81d8fdd9d4#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:02,490 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/02c4186e146e46c78b28a90d1109603d is 50, key is test_row_0/C:col10/1731230281054/Put/seqid=0 2024-11-10T09:18:02,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742279_1455 (size=13153) 2024-11-10T09:18:02,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/cbaa854a71464df8ab7fc86582f8c467 is 50, key is test_row_0/B:col10/1731230281458/Put/seqid=0 2024-11-10T09:18:02,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742280_1456 (size=12301) 2024-11-10T09:18:02,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. as already flushing 2024-11-10T09:18:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230342658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230342666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230342672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230342778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230342778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,804 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/4ef488e18e024fd3b55d72f0fe69a646 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/4ef488e18e024fd3b55d72f0fe69a646 2024-11-10T09:18:02,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230342799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:02,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:02,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:02,809 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/B of 32f231a2c762c9b2ac161a81d8fdd9d4 into 4ef488e18e024fd3b55d72f0fe69a646(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:02,809 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:02,809 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/B, priority=13, startTime=1731230282353; duration=0sec 2024-11-10T09:18:02,809 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:02,810 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:18:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:18:02,917 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/02c4186e146e46c78b28a90d1109603d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/02c4186e146e46c78b28a90d1109603d 2024-11-10T09:18:02,920 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 32f231a2c762c9b2ac161a81d8fdd9d4/C of 32f231a2c762c9b2ac161a81d8fdd9d4 into 02c4186e146e46c78b28a90d1109603d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:02,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:02,920 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., storeName=32f231a2c762c9b2ac161a81d8fdd9d4/C, priority=13, startTime=1731230282354; duration=0sec 2024-11-10T09:18:02,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:02,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:18:02,938 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/cbaa854a71464df8ab7fc86582f8c467 2024-11-10T09:18:02,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/ac0d1efa71084edbb807d52219c27ae5 is 50, key is test_row_0/C:col10/1731230281458/Put/seqid=0 2024-11-10T09:18:02,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742281_1457 (size=12301) 2024-11-10T09:18:03,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230342997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:03,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230342997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:03,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230343017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:03,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57086 deadline: 1731230343307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:03,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57072 deadline: 1731230343317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731230343327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:03,349 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/ac0d1efa71084edbb807d52219c27ae5 2024-11-10T09:18:03,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/298f90a44cc8440ab7082e51ae700a15 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/298f90a44cc8440ab7082e51ae700a15 2024-11-10T09:18:03,356 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/298f90a44cc8440ab7082e51ae700a15, entries=150, sequenceid=406, filesize=30.5 K 2024-11-10T09:18:03,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/cbaa854a71464df8ab7fc86582f8c467 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/cbaa854a71464df8ab7fc86582f8c467 2024-11-10T09:18:03,363 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/cbaa854a71464df8ab7fc86582f8c467, entries=150, sequenceid=406, filesize=12.0 K 2024-11-10T09:18:03,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/ac0d1efa71084edbb807d52219c27ae5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/ac0d1efa71084edbb807d52219c27ae5 2024-11-10T09:18:03,379 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/ac0d1efa71084edbb807d52219c27ae5, entries=150, sequenceid=406, filesize=12.0 K 2024-11-10T09:18:03,379 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1001ms, sequenceid=406, compaction requested=false 2024-11-10T09:18:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T09:18:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T09:18:03,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T09:18:03,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2024-11-10T09:18:03,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6270 sec 2024-11-10T09:18:03,648 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-11-10T09:18:03,648 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-11-10T09:18:03,648 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,648 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,648 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,648 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,649 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,649 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,651 INFO [Thread-1600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1600. 2024-11-10T09:18:03,651 DEBUG [Thread-1600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,651 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,652 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,657 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-10T09:18:03,657 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,657 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,657 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,657 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-11-10T09:18:03,657 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,657 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,657 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:03,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T09:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:18:03,829 INFO [Thread-1591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1591. 2024-11-10T09:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:03,829 DEBUG [Thread-1591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,829 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:18:03,829 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:03,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111050341eb787a04fa6afa3b718057248b0_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230283827/Put/seqid=0 2024-11-10T09:18:03,839 INFO [Thread-1587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1587. 2024-11-10T09:18:03,839 DEBUG [Thread-1587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,839 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,839 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742282_1458 (size=12454) 2024-11-10T09:18:03,841 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:03,844 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111050341eb787a04fa6afa3b718057248b0_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111050341eb787a04fa6afa3b718057248b0_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:03,844 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/a150f10512344b35b1d69722573d5b5c, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:03,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/a150f10512344b35b1d69722573d5b5c is 175, key is test_row_0/A:col10/1731230283827/Put/seqid=0 2024-11-10T09:18:03,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742283_1459 (size=31255) 2024-11-10T09:18:03,849 INFO [Thread-1593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1593. 2024-11-10T09:18:03,849 DEBUG [Thread-1593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,849 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,849 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,899 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-10T09:18:03,899 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,899 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,899 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:18:03,908 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:03,999 INFO [Thread-1589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1589. 2024-11-10T09:18:03,999 DEBUG [Thread-1589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:03,999 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,999 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:03,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:03,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:18:03,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-10T09:18:03,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-10T09:18:03,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-10T09:18:03,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 766 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2297 rows 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 764 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2292 rows 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 768 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2301 rows 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 767 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2288 rows 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 777 2024-11-10T09:18:04,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2320 rows 2024-11-10T09:18:04,000 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:18:04,000 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:18:04,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:04,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:04,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:04,001 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:18:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:04,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230284004"}]},"ts":"1731230284004"} 2024-11-10T09:18:04,005 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:18:04,005 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:18:04,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:18:04,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, UNASSIGN}] 2024-11-10T09:18:04,007 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, UNASSIGN 2024-11-10T09:18:04,007 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:04,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, UNASSIGN because future has completed 2024-11-10T09:18:04,009 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:18:04,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:18:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:04,161 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:04,161 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:18:04,161 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 32f231a2c762c9b2ac161a81d8fdd9d4, disabling compactions & flushes 2024-11-10T09:18:04,161 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:04,248 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/a150f10512344b35b1d69722573d5b5c 2024-11-10T09:18:04,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/c6a83b43afec42109f1e20ef8b3db358 is 50, key is test_row_0/B:col10/1731230283827/Put/seqid=0 2024-11-10T09:18:04,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742284_1460 (size=12301) 2024-11-10T09:18:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:04,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/c6a83b43afec42109f1e20ef8b3db358 2024-11-10T09:18:04,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/817bd26e6ec9463fbfc5f583c95e3376 is 50, key is test_row_0/C:col10/1731230283827/Put/seqid=0 2024-11-10T09:18:04,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742285_1461 (size=12301) 2024-11-10T09:18:05,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/817bd26e6ec9463fbfc5f583c95e3376 2024-11-10T09:18:05,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/a150f10512344b35b1d69722573d5b5c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/a150f10512344b35b1d69722573d5b5c 2024-11-10T09:18:05,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/a150f10512344b35b1d69722573d5b5c, entries=150, sequenceid=425, filesize=30.5 K 2024-11-10T09:18:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/c6a83b43afec42109f1e20ef8b3db358 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/c6a83b43afec42109f1e20ef8b3db358 2024-11-10T09:18:05,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/c6a83b43afec42109f1e20ef8b3db358, entries=150, sequenceid=425, filesize=12.0 K 2024-11-10T09:18:05,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/817bd26e6ec9463fbfc5f583c95e3376 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/817bd26e6ec9463fbfc5f583c95e3376 2024-11-10T09:18:05,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/817bd26e6ec9463fbfc5f583c95e3376, entries=150, sequenceid=425, filesize=12.0 K 2024-11-10T09:18:05,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1260ms, sequenceid=425, compaction requested=true 2024-11-10T09:18:05,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 32f231a2c762c9b2ac161a81d8fdd9d4: 2024-11-10T09:18:05,089 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:05,090 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. because compaction request was cancelled 2024-11-10T09:18:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. after waiting 0 ms 2024-11-10T09:18:05,090 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:A 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:05,090 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. because compaction request was cancelled 2024-11-10T09:18:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:B, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:05,090 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:B 2024-11-10T09:18:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:05,090 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 32f231a2c762c9b2ac161a81d8fdd9d4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T09:18:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 32f231a2c762c9b2ac161a81d8fdd9d4:C, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:05,090 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. because compaction request was cancelled 2024-11-10T09:18:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:05,090 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 32f231a2c762c9b2ac161a81d8fdd9d4:C 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=A 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=B 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 32f231a2c762c9b2ac161a81d8fdd9d4, store=C 2024-11-10T09:18:05,090 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:05,097 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101a9ec6ced1b84a36a67f82225c96c4ee_32f231a2c762c9b2ac161a81d8fdd9d4 is 50, key is test_row_0/A:col10/1731230283998/Put/seqid=0 2024-11-10T09:18:05,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742286_1462 (size=7374) 2024-11-10T09:18:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:05,502 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:05,505 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101a9ec6ced1b84a36a67f82225c96c4ee_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101a9ec6ced1b84a36a67f82225c96c4ee_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:05,505 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/3d38d208ea5341c7af6107c37902d2e1, store: [table=TestAcidGuarantees family=A region=32f231a2c762c9b2ac161a81d8fdd9d4] 2024-11-10T09:18:05,506 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/3d38d208ea5341c7af6107c37902d2e1 is 175, key is test_row_0/A:col10/1731230283998/Put/seqid=0 2024-11-10T09:18:05,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742287_1463 (size=13865) 2024-11-10T09:18:05,909 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=432, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/3d38d208ea5341c7af6107c37902d2e1 2024-11-10T09:18:05,915 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/3185ec8a4ec44920aac24397491353bb is 50, key is test_row_0/B:col10/1731230283998/Put/seqid=0 2024-11-10T09:18:05,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742288_1464 (size=7415) 2024-11-10T09:18:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:06,319 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/3185ec8a4ec44920aac24397491353bb 2024-11-10T09:18:06,324 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/6dad85b492e14f42b8bc8a798bcc5ae2 is 50, key is test_row_0/C:col10/1731230283998/Put/seqid=0 2024-11-10T09:18:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742289_1465 (size=7415) 2024-11-10T09:18:06,728 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/6dad85b492e14f42b8bc8a798bcc5ae2 2024-11-10T09:18:06,733 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/A/3d38d208ea5341c7af6107c37902d2e1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3d38d208ea5341c7af6107c37902d2e1 2024-11-10T09:18:06,736 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3d38d208ea5341c7af6107c37902d2e1, entries=50, sequenceid=432, filesize=13.5 K 2024-11-10T09:18:06,737 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/B/3185ec8a4ec44920aac24397491353bb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/3185ec8a4ec44920aac24397491353bb 2024-11-10T09:18:06,740 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/3185ec8a4ec44920aac24397491353bb, entries=50, sequenceid=432, filesize=7.2 K 2024-11-10T09:18:06,741 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/.tmp/C/6dad85b492e14f42b8bc8a798bcc5ae2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/6dad85b492e14f42b8bc8a798bcc5ae2 2024-11-10T09:18:06,745 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/6dad85b492e14f42b8bc8a798bcc5ae2, entries=50, sequenceid=432, filesize=7.2 K 2024-11-10T09:18:06,745 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1655ms, sequenceid=432, compaction requested=true 2024-11-10T09:18:06,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/6b22478a855442bdb1fad2fa904986b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/b3384916ea534f3398dfcc108716f705, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/83c83589ff4446daaa228be9a7baa0c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d17eabb43da4f00bbca827cba84d0e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3409a074f7a14fde8fed34d217380eaf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/faca53c3d48d45aba5cd6f698aadc057, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/13a0a8687a854225b6239af3c50c6107, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72] to archive 2024-11-10T09:18:06,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:18:06,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/2e2f81dce0a44db88e028073338ed7f9 2024-11-10T09:18:06,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c1f1123573144c8f99463229edbcfc34 2024-11-10T09:18:06,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/5ab71ef88fe746458f4dade5ca1677a2 2024-11-10T09:18:06,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/6b22478a855442bdb1fad2fa904986b1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/6b22478a855442bdb1fad2fa904986b1 2024-11-10T09:18:06,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0f1b349bf0084c2596a2981cfb22e8d2 2024-11-10T09:18:06,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/735ee2424ad642b1980c5703c66536ac 2024-11-10T09:18:06,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/130a8320022b4b0a96ace8dfe739105e 2024-11-10T09:18:06,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/b3384916ea534f3398dfcc108716f705 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/b3384916ea534f3398dfcc108716f705 2024-11-10T09:18:06,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/471e0dde4bb54619bba562fd7b89de20 2024-11-10T09:18:06,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/bb9e49de1cec4bb688fd90ea02c5311f 2024-11-10T09:18:06,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/83c83589ff4446daaa228be9a7baa0c0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/83c83589ff4446daaa228be9a7baa0c0 2024-11-10T09:18:06,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0a6103f284984df9b5951ddf73f3530a 2024-11-10T09:18:06,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/fdfe93f22698486c89f249c19a8797e0 2024-11-10T09:18:06,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d17eabb43da4f00bbca827cba84d0e0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d17eabb43da4f00bbca827cba84d0e0 2024-11-10T09:18:06,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/8d745a2f81e047ffa9a59bd37169c38c 2024-11-10T09:18:06,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/138d84ca3da942b7be6cf0da967e8ec8 2024-11-10T09:18:06,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3409a074f7a14fde8fed34d217380eaf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3409a074f7a14fde8fed34d217380eaf 2024-11-10T09:18:06,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/db0a47e3db394f4082cf9d265e9bc953 2024-11-10T09:18:06,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/0ca55e58014e4a0a9d72a1aac775ae0c 2024-11-10T09:18:06,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/faca53c3d48d45aba5cd6f698aadc057 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/faca53c3d48d45aba5cd6f698aadc057 2024-11-10T09:18:06,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/979f33f486da4c9da2517af8270953ee 2024-11-10T09:18:06,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/11c8ca086ad44edeba0fe1f4ec88a675 2024-11-10T09:18:06,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/13a0a8687a854225b6239af3c50c6107 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/13a0a8687a854225b6239af3c50c6107 2024-11-10T09:18:06,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/f0f4c9cb83af4a20bf5306e09a4369e2 2024-11-10T09:18:06,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/c113dc39655a458193f24d5143561d72 2024-11-10T09:18:06,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a9dae62afd92415093852a3fa9fff77d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f646825aef78409c8981fdf79885a54c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f1fb6f7ce46a48fea8137c475411db97, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/b41861a665234d0bbc17c7e8ae0c7ecb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/55c0ea81bf44437b875ecdd92d3b7126, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a832e81e3c4241cda1d87c9653bf3785, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/74e7895acb9e402eae08451ab0833c34, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5377855efd4344c2a91a06bd02c4ddc5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ec0d0eca843143a7bbbf4c6cebb95254, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a81cb04023d94387938da955cafccd3d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a5172869928a4cf2967608f01e232cd3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/29e460fc152c4437943b6cd3e4ba52ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/54e41d90322e4d0598b95a2ff9696e3a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8c8ad4a262024a0dbdcbfbe938fe4ca3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/db3f6c7598994da880d036accb7dd92b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ffef22d7380a46289854e6fcfea10cd9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/d67555b9de4945b7b6838d68cdf0b46f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/78d863fe63214b05afff6b5afb3473b1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/374b045e25684b55ab29db8765e2b0ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/62d50ae902424abd8da7073e137e1a38, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5474217591da44a99db423f20ce4dcfb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/15060185904e498eafd375f89559df06, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/eee0363c1b9247d3bcd00173bc41ef69, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8790f0dd81ee40d688b9535303e2a35e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5fb5c2c82f854624bab2ae2828d6bd43] to archive 2024-11-10T09:18:06,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:18:06,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a9dae62afd92415093852a3fa9fff77d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a9dae62afd92415093852a3fa9fff77d 2024-11-10T09:18:06,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f646825aef78409c8981fdf79885a54c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f646825aef78409c8981fdf79885a54c 2024-11-10T09:18:06,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f1fb6f7ce46a48fea8137c475411db97 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/f1fb6f7ce46a48fea8137c475411db97 2024-11-10T09:18:06,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/b41861a665234d0bbc17c7e8ae0c7ecb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/b41861a665234d0bbc17c7e8ae0c7ecb 2024-11-10T09:18:06,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/55c0ea81bf44437b875ecdd92d3b7126 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/55c0ea81bf44437b875ecdd92d3b7126 2024-11-10T09:18:06,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a832e81e3c4241cda1d87c9653bf3785 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a832e81e3c4241cda1d87c9653bf3785 2024-11-10T09:18:06,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/74e7895acb9e402eae08451ab0833c34 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/74e7895acb9e402eae08451ab0833c34 2024-11-10T09:18:06,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5377855efd4344c2a91a06bd02c4ddc5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5377855efd4344c2a91a06bd02c4ddc5 2024-11-10T09:18:06,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ec0d0eca843143a7bbbf4c6cebb95254 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ec0d0eca843143a7bbbf4c6cebb95254 2024-11-10T09:18:06,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a81cb04023d94387938da955cafccd3d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a81cb04023d94387938da955cafccd3d 2024-11-10T09:18:06,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a5172869928a4cf2967608f01e232cd3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/a5172869928a4cf2967608f01e232cd3 2024-11-10T09:18:06,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/29e460fc152c4437943b6cd3e4ba52ed to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/29e460fc152c4437943b6cd3e4ba52ed 2024-11-10T09:18:06,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/54e41d90322e4d0598b95a2ff9696e3a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/54e41d90322e4d0598b95a2ff9696e3a 2024-11-10T09:18:06,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8c8ad4a262024a0dbdcbfbe938fe4ca3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8c8ad4a262024a0dbdcbfbe938fe4ca3 2024-11-10T09:18:06,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/db3f6c7598994da880d036accb7dd92b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/db3f6c7598994da880d036accb7dd92b 2024-11-10T09:18:06,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ffef22d7380a46289854e6fcfea10cd9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/ffef22d7380a46289854e6fcfea10cd9 2024-11-10T09:18:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/d67555b9de4945b7b6838d68cdf0b46f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/d67555b9de4945b7b6838d68cdf0b46f 2024-11-10T09:18:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/78d863fe63214b05afff6b5afb3473b1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/78d863fe63214b05afff6b5afb3473b1 2024-11-10T09:18:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/374b045e25684b55ab29db8765e2b0ed to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/374b045e25684b55ab29db8765e2b0ed 2024-11-10T09:18:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/62d50ae902424abd8da7073e137e1a38 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/62d50ae902424abd8da7073e137e1a38 2024-11-10T09:18:06,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5474217591da44a99db423f20ce4dcfb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5474217591da44a99db423f20ce4dcfb 2024-11-10T09:18:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/15060185904e498eafd375f89559df06 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/15060185904e498eafd375f89559df06 2024-11-10T09:18:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/eee0363c1b9247d3bcd00173bc41ef69 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/eee0363c1b9247d3bcd00173bc41ef69 2024-11-10T09:18:06,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8790f0dd81ee40d688b9535303e2a35e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/8790f0dd81ee40d688b9535303e2a35e 2024-11-10T09:18:06,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5fb5c2c82f854624bab2ae2828d6bd43 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/5fb5c2c82f854624bab2ae2828d6bd43 2024-11-10T09:18:06,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0985949fd22849c49bc452c7dec4ae64, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/b9e2639490724b0ab2dcab2cc970c92a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d533605489614e30be595e50e92749d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/987e4922486f4aa7baa7474f31a0342b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4642d9a3eba948e5baad6cd1cba65e56, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cf6973df617b4fb4bd0ce241c706dce0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/2f4adb079ae142c699a969ad68d160d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/07c380b6f7a2454fb4cecb4aeeb4d960, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f59250fe5ec445f88d5b7018245045ec, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/a7cefb22863441909360af0029e8066e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/8b307af7ed2e459d91a4040a06b4a9e4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/dab89e04f8334f6d8f14cbd207e6b77f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/7bd18704cc9141a19f1363ac4bac4e07, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0bf00d15eef144bbade3c468d4c66e1a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cfe89e79bd514770be083958c898fd06, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/5d7557a91bd3409fa9a8dc181eef793b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/725ab006c6bb4396bac0aed496ddc338, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/c667cad5a8a64c718db037dfa26c6e9e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f37b082423e24172987e381ca55188b5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/fc56aae91ccf447dbd5987bc42f32e98, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/12b14871beff4820a81ca19c73dc16d4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4f224be6938d4c94b55212befa9b915f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/356d95dbc65a43b9a69b6dc3dad048a5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/059a914bc8e94ac3b2dbb8165dc7e898] to archive 2024-11-10T09:18:06,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:18:06,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:18:06,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0985949fd22849c49bc452c7dec4ae64 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0985949fd22849c49bc452c7dec4ae64 2024-11-10T09:18:06,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/b9e2639490724b0ab2dcab2cc970c92a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/b9e2639490724b0ab2dcab2cc970c92a 2024-11-10T09:18:06,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d533605489614e30be595e50e92749d7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d533605489614e30be595e50e92749d7 2024-11-10T09:18:06,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/987e4922486f4aa7baa7474f31a0342b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/987e4922486f4aa7baa7474f31a0342b 2024-11-10T09:18:06,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4642d9a3eba948e5baad6cd1cba65e56 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4642d9a3eba948e5baad6cd1cba65e56 2024-11-10T09:18:06,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cf6973df617b4fb4bd0ce241c706dce0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cf6973df617b4fb4bd0ce241c706dce0 2024-11-10T09:18:06,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/2f4adb079ae142c699a969ad68d160d7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/2f4adb079ae142c699a969ad68d160d7 2024-11-10T09:18:06,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/07c380b6f7a2454fb4cecb4aeeb4d960 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/07c380b6f7a2454fb4cecb4aeeb4d960 2024-11-10T09:18:06,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/d1a8b0d5a98e46aabd9a2e7d3208f1e0 2024-11-10T09:18:06,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f59250fe5ec445f88d5b7018245045ec to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f59250fe5ec445f88d5b7018245045ec 2024-11-10T09:18:06,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/a7cefb22863441909360af0029e8066e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/a7cefb22863441909360af0029e8066e 2024-11-10T09:18:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/8b307af7ed2e459d91a4040a06b4a9e4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/8b307af7ed2e459d91a4040a06b4a9e4 2024-11-10T09:18:06,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/dab89e04f8334f6d8f14cbd207e6b77f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/dab89e04f8334f6d8f14cbd207e6b77f 2024-11-10T09:18:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/7bd18704cc9141a19f1363ac4bac4e07 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/7bd18704cc9141a19f1363ac4bac4e07 2024-11-10T09:18:06,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0bf00d15eef144bbade3c468d4c66e1a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/0bf00d15eef144bbade3c468d4c66e1a 2024-11-10T09:18:06,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cfe89e79bd514770be083958c898fd06 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/cfe89e79bd514770be083958c898fd06 2024-11-10T09:18:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/5d7557a91bd3409fa9a8dc181eef793b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/5d7557a91bd3409fa9a8dc181eef793b 2024-11-10T09:18:06,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/725ab006c6bb4396bac0aed496ddc338 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/725ab006c6bb4396bac0aed496ddc338 2024-11-10T09:18:06,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/c667cad5a8a64c718db037dfa26c6e9e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/c667cad5a8a64c718db037dfa26c6e9e 2024-11-10T09:18:06,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f37b082423e24172987e381ca55188b5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/f37b082423e24172987e381ca55188b5 2024-11-10T09:18:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/fc56aae91ccf447dbd5987bc42f32e98 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/fc56aae91ccf447dbd5987bc42f32e98 2024-11-10T09:18:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/12b14871beff4820a81ca19c73dc16d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/12b14871beff4820a81ca19c73dc16d4 2024-11-10T09:18:06,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4f224be6938d4c94b55212befa9b915f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/4f224be6938d4c94b55212befa9b915f 2024-11-10T09:18:06,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/356d95dbc65a43b9a69b6dc3dad048a5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/356d95dbc65a43b9a69b6dc3dad048a5 2024-11-10T09:18:06,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/059a914bc8e94ac3b2dbb8165dc7e898 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/059a914bc8e94ac3b2dbb8165dc7e898 2024-11-10T09:18:06,826 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/recovered.edits/435.seqid, newMaxSeqId=435, maxSeqId=4 2024-11-10T09:18:06,827 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. 2024-11-10T09:18:06,827 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 32f231a2c762c9b2ac161a81d8fdd9d4: Waiting for close lock at 1731230284161Running coprocessor pre-close hooks at 1731230284161Disabling compacts and flushes for region at 1731230284161Disabling writes for close at 1731230285090 (+929 ms)Obtaining lock to block concurrent updates at 1731230285090Preparing flush snapshotting stores in 32f231a2c762c9b2ac161a81d8fdd9d4 at 1731230285090Finished memstore snapshotting TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731230285090Flushing stores of TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4. at 1731230285091 (+1 ms)Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/A: creating writer at 1731230285091Flushing mob file 32f231a2c762c9b2ac161a81d8fdd9d4/A: appending metadata at 1731230285097 (+6 ms)Flushing mob file 32f231a2c762c9b2ac161a81d8fdd9d4/A: closing flushed file at 1731230285097Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/A: appending metadata at 1731230285505 (+408 ms)Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/A: closing flushed file at 1731230285505Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/B: creating writer at 1731230285912 (+407 ms)Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/B: appending metadata at 1731230285915 (+3 ms)Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/B: closing flushed file at 1731230285915Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/C: creating writer at 1731230286322 (+407 ms)Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/C: appending metadata at 1731230286324 (+2 ms)Flushing 32f231a2c762c9b2ac161a81d8fdd9d4/C: closing flushed file at 1731230286324Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@258d92b6: reopening flushed file at 1731230286733 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30517032: reopening flushed file at 1731230286736 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bc759ea: reopening flushed file at 1731230286740 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 32f231a2c762c9b2ac161a81d8fdd9d4 in 1655ms, sequenceid=432, compaction requested=true at 1731230286745 (+5 ms)Writing region close event to WAL at 1731230286824 (+79 ms)Running coprocessor post-close hooks at 1731230286827 (+3 ms)Closed at 1731230286827 2024-11-10T09:18:06,829 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:06,829 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=32f231a2c762c9b2ac161a81d8fdd9d4, regionState=CLOSED 2024-11-10T09:18:06,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:18:06,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T09:18:06,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 32f231a2c762c9b2ac161a81d8fdd9d4, server=802346a334f5,38513,1731230168538 in 2.8220 sec 2024-11-10T09:18:06,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-10T09:18:06,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=32f231a2c762c9b2ac161a81d8fdd9d4, UNASSIGN in 2.8260 sec 2024-11-10T09:18:06,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T09:18:06,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8300 sec 2024-11-10T09:18:06,837 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230286837"}]},"ts":"1731230286837"} 2024-11-10T09:18:06,838 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:18:06,838 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:18:06,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8380 sec 2024-11-10T09:18:08,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:18:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:18:08,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:08,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:18:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,169 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:18:08,170 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,172 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,173 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/recovered.edits] 2024-11-10T09:18:08,176 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/298f90a44cc8440ab7082e51ae700a15 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/298f90a44cc8440ab7082e51ae700a15 2024-11-10T09:18:08,177 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3d38d208ea5341c7af6107c37902d2e1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/3d38d208ea5341c7af6107c37902d2e1 2024-11-10T09:18:08,178 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/631a6b282e7047d48e881d7607dc5f5e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/631a6b282e7047d48e881d7607dc5f5e 2024-11-10T09:18:08,179 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/a150f10512344b35b1d69722573d5b5c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/A/a150f10512344b35b1d69722573d5b5c 2024-11-10T09:18:08,181 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/3185ec8a4ec44920aac24397491353bb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/3185ec8a4ec44920aac24397491353bb 2024-11-10T09:18:08,181 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/4ef488e18e024fd3b55d72f0fe69a646 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/4ef488e18e024fd3b55d72f0fe69a646 2024-11-10T09:18:08,182 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/c6a83b43afec42109f1e20ef8b3db358 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/c6a83b43afec42109f1e20ef8b3db358 2024-11-10T09:18:08,183 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/cbaa854a71464df8ab7fc86582f8c467 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/B/cbaa854a71464df8ab7fc86582f8c467 2024-11-10T09:18:08,185 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/02c4186e146e46c78b28a90d1109603d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/02c4186e146e46c78b28a90d1109603d 2024-11-10T09:18:08,186 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/6dad85b492e14f42b8bc8a798bcc5ae2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/6dad85b492e14f42b8bc8a798bcc5ae2 2024-11-10T09:18:08,187 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/817bd26e6ec9463fbfc5f583c95e3376 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/817bd26e6ec9463fbfc5f583c95e3376 2024-11-10T09:18:08,187 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/ac0d1efa71084edbb807d52219c27ae5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/C/ac0d1efa71084edbb807d52219c27ae5 2024-11-10T09:18:08,190 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/recovered.edits/435.seqid to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4/recovered.edits/435.seqid 2024-11-10T09:18:08,190 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,190 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:18:08,190 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:18:08,191 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T09:18:08,193 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c22209eddef44b6b4def8ec99d26697_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c22209eddef44b6b4def8ec99d26697_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,194 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c35f91d7cf34c89b5fb73c14114a6d6_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c35f91d7cf34c89b5fb73c14114a6d6_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,195 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101374dcf4e96747759a97cde610102b2b_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101374dcf4e96747759a97cde610102b2b_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,196 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014040862d51d43479ccf1afaa61d8bb9_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014040862d51d43479ccf1afaa61d8bb9_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,197 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101a9ec6ced1b84a36a67f82225c96c4ee_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101a9ec6ced1b84a36a67f82225c96c4ee_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,197 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e48cc08fba4416d8cee6e7bedade6c7_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e48cc08fba4416d8cee6e7bedade6c7_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,198 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e6f27aef5f44fa8ad33303fe7d0d638_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e6f27aef5f44fa8ad33303fe7d0d638_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,199 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103ef61b769880450ab56aaf6f70b808df_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103ef61b769880450ab56aaf6f70b808df_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,199 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111050341eb787a04fa6afa3b718057248b0_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111050341eb787a04fa6afa3b718057248b0_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,200 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051ce7aaac9f440598bc6b7fbfd31f6ed_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051ce7aaac9f440598bc6b7fbfd31f6ed_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,201 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076ecaf66285146dca88fb702421bd3ac_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076ecaf66285146dca88fb702421bd3ac_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,202 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108db0a4a7224f423c830ff28e9915a056_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108db0a4a7224f423c830ff28e9915a056_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,203 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108e6fde44bffe4d7998c8fac46d131e9a_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108e6fde44bffe4d7998c8fac46d131e9a_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,204 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110957207fe3d2446b9aee6b14944c476db_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110957207fe3d2446b9aee6b14944c476db_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,205 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b3a669b039ea48319101cec3fa6f7abf_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b3a669b039ea48319101cec3fa6f7abf_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,206 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b6f7946f085744b9963c79217e08d46a_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b6f7946f085744b9963c79217e08d46a_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,207 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bd8b1417569b4c07b8c3eab9c8ced5a2_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bd8b1417569b4c07b8c3eab9c8ced5a2_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,207 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cf87522e25d048189c83822fd99f44bd_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cf87522e25d048189c83822fd99f44bd_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,208 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d041243601684ceeaf70a98acd92c18f_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d041243601684ceeaf70a98acd92c18f_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,209 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8d703e3e1cc44228b84f3a5464d25ac_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8d703e3e1cc44228b84f3a5464d25ac_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,210 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9777150ace24bcbae6ef16a096f37d2_32f231a2c762c9b2ac161a81d8fdd9d4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9777150ace24bcbae6ef16a096f37d2_32f231a2c762c9b2ac161a81d8fdd9d4 2024-11-10T09:18:08,211 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:18:08,213 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,216 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:18:08,218 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:18:08,219 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,219 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:18:08,219 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230288219"}]},"ts":"9223372036854775807"} 2024-11-10T09:18:08,221 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:18:08,221 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 32f231a2c762c9b2ac161a81d8fdd9d4, NAME => 'TestAcidGuarantees,,1731230262014.32f231a2c762c9b2ac161a81d8fdd9d4.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:18:08,222 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:18:08,222 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230288222"}]},"ts":"9223372036854775807"} 2024-11-10T09:18:08,224 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:18:08,224 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-10T09:18:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:18:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:18:08,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:08,287 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=461 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=796 (was 737) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2387 (was 1191) - AvailableMemoryMB LEAK? - 2024-11-10T09:18:08,294 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=796, ProcessCount=11, AvailableMemoryMB=2387 2024-11-10T09:18:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:18:08,296 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:18:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:08,297 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:18:08,297 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:08,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-10T09:18:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-10T09:18:08,298 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:18:08,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742290_1466 (size=1061) 2024-11-10T09:18:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-10T09:18:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-10T09:18:08,704 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:18:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742291_1467 (size=53) 2024-11-10T09:18:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-10T09:18:09,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:18:09,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5b0c14639876e70a158c7f1b205d25b6, disabling compactions & flushes 2024-11-10T09:18:09,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. after waiting 0 ms 2024-11-10T09:18:09,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b0c14639876e70a158c7f1b205d25b6: Waiting for close lock at 1731230289121Disabling compacts and flushes for region at 1731230289121Disabling writes for close at 1731230289121Writing region close event to WAL at 1731230289121Closed at 1731230289121 2024-11-10T09:18:09,122 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:18:09,122 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230289122"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230289122"}]},"ts":"1731230289122"} 2024-11-10T09:18:09,124 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:18:09,125 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:18:09,125 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230289125"}]},"ts":"1731230289125"} 2024-11-10T09:18:09,127 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:18:09,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, ASSIGN}] 2024-11-10T09:18:09,129 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, ASSIGN 2024-11-10T09:18:09,129 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, ASSIGN; state=OFFLINE, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=false 2024-11-10T09:18:09,280 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=5b0c14639876e70a158c7f1b205d25b6, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:09,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, ASSIGN because future has completed 2024-11-10T09:18:09,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-10T09:18:09,438 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,439 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:18:09,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:18:09,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,441 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,442 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:09,443 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b0c14639876e70a158c7f1b205d25b6 columnFamilyName A 2024-11-10T09:18:09,443 DEBUG [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:09,443 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.HStore(327): Store=5b0c14639876e70a158c7f1b205d25b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:09,443 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,444 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:09,445 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b0c14639876e70a158c7f1b205d25b6 columnFamilyName B 2024-11-10T09:18:09,445 DEBUG [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:09,445 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.HStore(327): Store=5b0c14639876e70a158c7f1b205d25b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:09,445 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,446 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:09,446 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b0c14639876e70a158c7f1b205d25b6 columnFamilyName C 2024-11-10T09:18:09,446 DEBUG [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:09,447 INFO [StoreOpener-5b0c14639876e70a158c7f1b205d25b6-1 {}] regionserver.HStore(327): Store=5b0c14639876e70a158c7f1b205d25b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:09,447 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,447 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,448 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,448 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,449 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,449 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,450 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:18:09,451 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,452 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:18:09,452 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 5b0c14639876e70a158c7f1b205d25b6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74998984, jitterRate=0.11757194995880127}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:18:09,452 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:09,453 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 5b0c14639876e70a158c7f1b205d25b6: Running coprocessor pre-open hook at 1731230289440Writing region info on filesystem at 1731230289440Initializing all the Stores at 1731230289441 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230289441Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230289441Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230289441Cleaning up temporary data from old regions at 1731230289449 (+8 ms)Running coprocessor post-open hooks at 1731230289452 (+3 ms)Region opened successfully at 1731230289453 (+1 ms) 2024-11-10T09:18:09,454 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., pid=118, masterSystemTime=1731230289435 2024-11-10T09:18:09,455 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,455 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:09,455 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=5b0c14639876e70a158c7f1b205d25b6, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:09,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:18:09,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T09:18:09,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 in 173 msec 2024-11-10T09:18:09,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-10T09:18:09,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, ASSIGN in 332 msec 2024-11-10T09:18:09,461 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:18:09,461 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230289461"}]},"ts":"1731230289461"} 2024-11-10T09:18:09,463 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:18:09,463 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:18:09,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-10T09:18:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-10T09:18:10,449 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:10,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec5b892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,452 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb6c7b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6463b671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,455 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,458 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:18:10,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:18:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@216ca96a 2024-11-10T09:18:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:18:10,460 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:18:10,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250ddf0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,462 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ada0a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,463 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6949e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc2be37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,465 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c575735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,468 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d385079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c47751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,471 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc740fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,473 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42abcf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e1154a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,474 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5e3aac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,476 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ac8f67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79925993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,477 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5f5afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae44c29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19aa15e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,480 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3a6efc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7afef795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3cd49e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,482 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302e6f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26034fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b78dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,485 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14277c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,487 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ea1c16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c5ed93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4964c112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281c012f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c06257c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:10,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:10,495 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bc6902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:10,497 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502a5972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:10,498 DEBUG [Thread-2088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,498 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,498 DEBUG [Thread-2099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,500 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,501 DEBUG [Thread-2101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,502 DEBUG [Thread-2086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,503 DEBUG [Thread-2090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,504 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,505 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T09:18:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,508 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:10,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:10,514 DEBUG [Thread-2097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:10,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2] 2024-11-10T09:18:10,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:18:10,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:10,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:10,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:10,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:10,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:10,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:10,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/e338d06988064b49860390eba1f417da is 50, key is test_row_0/A:col10/1731230290509/Put/seqid=0 2024-11-10T09:18:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742292_1468 (size=12001) 2024-11-10T09:18:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230350559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230350559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230350561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230290535, max timestamp=1731230290572Num uniques 859; 2024-11-10T09:18:10,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230290535, max timestamp=1731230290572Num uniques 859; 2024-11-10T09:18:10,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:10,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:10,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230290535, max timestamp=1731230290572Num uniques 859; 2024-11-10T09:18:10,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230350628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230350632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:10,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230350678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230350678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230350689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230350737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230350747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:10,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230350897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230350897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230350897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230350947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:10,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/e338d06988064b49860390eba1f417da 2024-11-10T09:18:10,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:10,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230350967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/50e95f60218d455fa84ebab90541b101 is 50, key is test_row_0/B:col10/1731230290509/Put/seqid=0 2024-11-10T09:18:11,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742293_1469 (size=12001) 2024-11-10T09:18:11,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:11,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230351217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230351217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230351218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230351267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230351288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/50e95f60218d455fa84ebab90541b101 2024-11-10T09:18:11,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/37a7373395144e63a4b15505926879aa is 50, key is test_row_0/C:col10/1731230290509/Put/seqid=0 2024-11-10T09:18:11,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742294_1470 (size=12001) 2024-11-10T09:18:11,542 WARN [regionserver/802346a334f5:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 1 2024-11-10T09:18:11,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230351737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230351737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230351737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:11,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:11,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230351787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230351807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:11,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/37a7373395144e63a4b15505926879aa 2024-11-10T09:18:11,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/e338d06988064b49860390eba1f417da as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e338d06988064b49860390eba1f417da 2024-11-10T09:18:11,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e338d06988064b49860390eba1f417da, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:18:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/50e95f60218d455fa84ebab90541b101 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50e95f60218d455fa84ebab90541b101 2024-11-10T09:18:11,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50e95f60218d455fa84ebab90541b101, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:18:11,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/37a7373395144e63a4b15505926879aa as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/37a7373395144e63a4b15505926879aa 2024-11-10T09:18:11,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/37a7373395144e63a4b15505926879aa, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:18:11,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 5b0c14639876e70a158c7f1b205d25b6 in 1360ms, sequenceid=13, compaction requested=false 2024-11-10T09:18:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:11,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T09:18:11,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:11,898 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-10T09:18:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:11,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/5d3aeffc27b746d2a8c60264ee1f8d5f is 50, key is test_row_0/A:col10/1731230290619/Put/seqid=0 2024-11-10T09:18:11,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742295_1471 (size=12001) 2024-11-10T09:18:12,310 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/5d3aeffc27b746d2a8c60264ee1f8d5f 2024-11-10T09:18:12,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/ebf851be1536407a874f8ba161a1ef6f is 50, key is test_row_0/B:col10/1731230290619/Put/seqid=0 2024-11-10T09:18:12,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742296_1472 (size=12001) 2024-11-10T09:18:12,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:18:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:12,720 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/ebf851be1536407a874f8ba161a1ef6f 2024-11-10T09:18:12,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/213849e1625d416691bc2fc9757f550d is 50, key is test_row_0/C:col10/1731230290619/Put/seqid=0 2024-11-10T09:18:12,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742297_1473 (size=12001) 2024-11-10T09:18:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:12,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230352793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230352795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230352798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230352807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,830 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230352827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,835 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230352907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230352907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:12,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230352917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230353117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230353118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,131 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/213849e1625d416691bc2fc9757f550d 2024-11-10T09:18:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/5d3aeffc27b746d2a8c60264ee1f8d5f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/5d3aeffc27b746d2a8c60264ee1f8d5f 2024-11-10T09:18:13,139 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/5d3aeffc27b746d2a8c60264ee1f8d5f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-10T09:18:13,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/ebf851be1536407a874f8ba161a1ef6f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ebf851be1536407a874f8ba161a1ef6f 2024-11-10T09:18:13,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230353137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,146 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ebf851be1536407a874f8ba161a1ef6f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-10T09:18:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/213849e1625d416691bc2fc9757f550d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/213849e1625d416691bc2fc9757f550d 2024-11-10T09:18:13,149 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/213849e1625d416691bc2fc9757f550d, entries=150, sequenceid=47, filesize=11.7 K 2024-11-10T09:18:13,149 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 5b0c14639876e70a158c7f1b205d25b6 in 1252ms, sequenceid=47, compaction requested=false 2024-11-10T09:18:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T09:18:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T09:18:13,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T09:18:13,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-11-10T09:18:13,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.6490 sec 2024-11-10T09:18:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:13,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:18:13,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:13,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/9bffaffdbf3f415f943aec97e5ed76e5 is 50, key is test_row_0/A:col10/1731230293441/Put/seqid=0 2024-11-10T09:18:13,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742298_1474 (size=14341) 2024-11-10T09:18:13,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/9bffaffdbf3f415f943aec97e5ed76e5 2024-11-10T09:18:13,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/5fdf7f9a59564b6ca093bf3a4a3c986c is 50, key is test_row_0/B:col10/1731230293441/Put/seqid=0 2024-11-10T09:18:13,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742299_1475 (size=12001) 2024-11-10T09:18:13,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230353521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230353522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230353528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230353637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230353637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230353637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230353858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230353858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230353858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:13,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/5fdf7f9a59564b6ca093bf3a4a3c986c 2024-11-10T09:18:13,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/cd1141c4e3494df28562f3b340276146 is 50, key is test_row_0/C:col10/1731230293441/Put/seqid=0 2024-11-10T09:18:13,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742300_1476 (size=12001) 2024-11-10T09:18:14,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230354177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230354177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230354177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/cd1141c4e3494df28562f3b340276146 2024-11-10T09:18:14,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/9bffaffdbf3f415f943aec97e5ed76e5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9bffaffdbf3f415f943aec97e5ed76e5 2024-11-10T09:18:14,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9bffaffdbf3f415f943aec97e5ed76e5, entries=200, sequenceid=60, filesize=14.0 K 2024-11-10T09:18:14,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/5fdf7f9a59564b6ca093bf3a4a3c986c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5fdf7f9a59564b6ca093bf3a4a3c986c 2024-11-10T09:18:14,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5fdf7f9a59564b6ca093bf3a4a3c986c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T09:18:14,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/cd1141c4e3494df28562f3b340276146 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/cd1141c4e3494df28562f3b340276146 2024-11-10T09:18:14,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/cd1141c4e3494df28562f3b340276146, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T09:18:14,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5b0c14639876e70a158c7f1b205d25b6 in 918ms, sequenceid=60, compaction requested=true 2024-11-10T09:18:14,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:14,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:14,361 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:14,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:14,361 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:14,362 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:14,362 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:14,362 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:14,362 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:14,362 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,362 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,363 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e338d06988064b49860390eba1f417da, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/5d3aeffc27b746d2a8c60264ee1f8d5f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9bffaffdbf3f415f943aec97e5ed76e5] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=37.4 K 2024-11-10T09:18:14,363 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50e95f60218d455fa84ebab90541b101, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ebf851be1536407a874f8ba161a1ef6f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5fdf7f9a59564b6ca093bf3a4a3c986c] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=35.2 K 2024-11-10T09:18:14,363 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting e338d06988064b49860390eba1f417da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230290509 2024-11-10T09:18:14,363 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e95f60218d455fa84ebab90541b101, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230290509 2024-11-10T09:18:14,363 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf851be1536407a874f8ba161a1ef6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731230290611 2024-11-10T09:18:14,363 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3aeffc27b746d2a8c60264ee1f8d5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731230290611 2024-11-10T09:18:14,363 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fdf7f9a59564b6ca093bf3a4a3c986c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230292773 2024-11-10T09:18:14,363 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bffaffdbf3f415f943aec97e5ed76e5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230292773 2024-11-10T09:18:14,385 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:14,386 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/50a45725a4374aad8d5c44ced379089b is 50, key is test_row_0/B:col10/1731230293441/Put/seqid=0 2024-11-10T09:18:14,423 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:14,423 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/9492117dd4c849a6b280213a37021168 is 50, key is test_row_0/A:col10/1731230293441/Put/seqid=0 2024-11-10T09:18:14,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742301_1477 (size=12104) 2024-11-10T09:18:14,431 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/50a45725a4374aad8d5c44ced379089b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50a45725a4374aad8d5c44ced379089b 2024-11-10T09:18:14,440 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into 50a45725a4374aad8d5c44ced379089b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:14,440 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:14,440 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=13, startTime=1731230294361; duration=0sec 2024-11-10T09:18:14,440 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:14,440 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:14,440 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:14,442 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:14,442 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:14,442 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,442 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/37a7373395144e63a4b15505926879aa, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/213849e1625d416691bc2fc9757f550d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/cd1141c4e3494df28562f3b340276146] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=35.2 K 2024-11-10T09:18:14,442 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a7373395144e63a4b15505926879aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230290509 2024-11-10T09:18:14,442 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 213849e1625d416691bc2fc9757f550d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731230290611 2024-11-10T09:18:14,443 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1141c4e3494df28562f3b340276146, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230292773 2024-11-10T09:18:14,449 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:14,450 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/e14cb508336943d4a18b9106bc105768 is 50, key is test_row_0/C:col10/1731230293441/Put/seqid=0 2024-11-10T09:18:14,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742303_1479 (size=12104) 2024-11-10T09:18:14,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742302_1478 (size=12104) 2024-11-10T09:18:14,470 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/9492117dd4c849a6b280213a37021168 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9492117dd4c849a6b280213a37021168 2024-11-10T09:18:14,481 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into 9492117dd4c849a6b280213a37021168(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:14,481 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:14,481 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=13, startTime=1731230294361; duration=0sec 2024-11-10T09:18:14,481 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:14,481 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T09:18:14,677 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:14,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T09:18:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:18:14,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:14,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:14,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:14,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:18:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:14,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/741b49eaf54f402ea663bb19db86e3a1 is 50, key is test_row_0/A:col10/1731230294701/Put/seqid=0 2024-11-10T09:18:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742304_1480 (size=14341) 2024-11-10T09:18:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230354744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230354745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230354758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:18:14,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:14,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230354847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,856 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230354857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,869 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,872 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/e14cb508336943d4a18b9106bc105768 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/e14cb508336943d4a18b9106bc105768 2024-11-10T09:18:14,876 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into e14cb508336943d4a18b9106bc105768(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:14,877 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:14,877 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=13, startTime=1731230294362; duration=0sec 2024-11-10T09:18:14,877 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:14,877 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:14,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230354868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230354869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230354877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:14,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:14,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:14,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:14,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:18:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230355087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230355087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230355097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:15,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/741b49eaf54f402ea663bb19db86e3a1 2024-11-10T09:18:15,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/323f574a9d284f35bbf94c5620f7cd90 is 50, key is test_row_0/B:col10/1731230294701/Put/seqid=0 2024-11-10T09:18:15,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742305_1481 (size=12001) 2024-11-10T09:18:15,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/323f574a9d284f35bbf94c5620f7cd90 2024-11-10T09:18:15,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/f8f76a4116a74831b1ee7f8844777a5f is 50, key is test_row_0/C:col10/1731230294701/Put/seqid=0 2024-11-10T09:18:15,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742306_1482 (size=12001) 2024-11-10T09:18:15,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:18:15,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230355407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:15,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230355407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:15,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230355417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:15,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:15,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:15,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/f8f76a4116a74831b1ee7f8844777a5f 2024-11-10T09:18:15,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/741b49eaf54f402ea663bb19db86e3a1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/741b49eaf54f402ea663bb19db86e3a1 2024-11-10T09:18:15,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/741b49eaf54f402ea663bb19db86e3a1, entries=200, sequenceid=82, filesize=14.0 K 2024-11-10T09:18:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/323f574a9d284f35bbf94c5620f7cd90 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/323f574a9d284f35bbf94c5620f7cd90 2024-11-10T09:18:15,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:15,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/323f574a9d284f35bbf94c5620f7cd90, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T09:18:15,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/f8f76a4116a74831b1ee7f8844777a5f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f8f76a4116a74831b1ee7f8844777a5f 2024-11-10T09:18:15,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f8f76a4116a74831b1ee7f8844777a5f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T09:18:15,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5b0c14639876e70a158c7f1b205d25b6 in 912ms, sequenceid=82, compaction requested=false 2024-11-10T09:18:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:15,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T09:18:15,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:15,759 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:18:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/1459f49f9208434b83196e4efdd1a2ae is 50, key is test_row_0/A:col10/1731230294744/Put/seqid=0 2024-11-10T09:18:15,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742307_1483 (size=12001) 2024-11-10T09:18:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:18:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:16,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230356024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230356024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230356025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230356139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230356146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230356148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,175 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/1459f49f9208434b83196e4efdd1a2ae 2024-11-10T09:18:16,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/a5fef21ac72348ac9a52d7582f6f7638 is 50, key is test_row_0/B:col10/1731230294744/Put/seqid=0 2024-11-10T09:18:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742308_1484 (size=12001) 2024-11-10T09:18:16,209 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/a5fef21ac72348ac9a52d7582f6f7638 2024-11-10T09:18:16,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/55cc37953757435e9ddd9cfe2ad930f1 is 50, key is test_row_0/C:col10/1731230294744/Put/seqid=0 2024-11-10T09:18:16,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742309_1485 (size=12001) 2024-11-10T09:18:16,244 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/55cc37953757435e9ddd9cfe2ad930f1 2024-11-10T09:18:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/1459f49f9208434b83196e4efdd1a2ae as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/1459f49f9208434b83196e4efdd1a2ae 2024-11-10T09:18:16,262 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/1459f49f9208434b83196e4efdd1a2ae, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T09:18:16,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/a5fef21ac72348ac9a52d7582f6f7638 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/a5fef21ac72348ac9a52d7582f6f7638 2024-11-10T09:18:16,273 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/a5fef21ac72348ac9a52d7582f6f7638, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T09:18:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/55cc37953757435e9ddd9cfe2ad930f1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/55cc37953757435e9ddd9cfe2ad930f1 2024-11-10T09:18:16,284 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/55cc37953757435e9ddd9cfe2ad930f1, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T09:18:16,285 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5b0c14639876e70a158c7f1b205d25b6 in 527ms, sequenceid=95, compaction requested=true 2024-11-10T09:18:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T09:18:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T09:18:16,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T09:18:16,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-11-10T09:18:16,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6160 sec 2024-11-10T09:18:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:16,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230295970, max timestamp=1731230296363Num uniques 859; 2024-11-10T09:18:16,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230295970, max timestamp=1731230296363Num uniques 859; 2024-11-10T09:18:16,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:16,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:16,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230295970, max timestamp=1731230296363Num uniques 859; 2024-11-10T09:18:16,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:16,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-10T09:18:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:16,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b0abdf2a87854979a485fda53dc5aea8 is 50, key is test_row_0/A:col10/1731230296023/Put/seqid=0 2024-11-10T09:18:16,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742310_1486 (size=12001) 2024-11-10T09:18:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230296414, max timestamp=1731230296433Num uniques 859; 2024-11-10T09:18:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230296414, max timestamp=1731230296433Num uniques 859; 2024-11-10T09:18:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230296414, max timestamp=1731230296433Num uniques 859; 2024-11-10T09:18:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:16,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731230296497, max timestamp=1731230296521Num uniques 1018; 2024-11-10T09:18:16,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731230296497, max timestamp=1731230296521Num uniques 1018; 2024-11-10T09:18:16,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T09:18:16,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T09:18:16,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731230296497, max timestamp=1731230296521Num uniques 1018; 2024-11-10T09:18:16,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T09:18:16,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731230296585, max timestamp=1731230296600Num uniques 1127; 2024-11-10T09:18:16,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-10T09:18:16,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731230296585, max timestamp=1731230296600Num uniques 1127; 2024-11-10T09:18:16,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-10T09:18:16,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731230296585, max timestamp=1731230296600Num uniques 1127; 2024-11-10T09:18:16,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-10T09:18:16,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731230296645, max timestamp=1731230296662Num uniques 1086; 2024-11-10T09:18:16,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T09:18:16,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731230296645, max timestamp=1731230296662Num uniques 1086; 2024-11-10T09:18:16,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T09:18:16,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731230296645, max timestamp=1731230296662Num uniques 1086; 2024-11-10T09:18:16,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T09:18:16,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731230296698, max timestamp=1731230296736Num uniques 1095; 2024-11-10T09:18:16,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T09:18:16,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731230296698, max timestamp=1731230296736Num uniques 1095; 2024-11-10T09:18:16,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T09:18:16,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731230296698, max timestamp=1731230296736Num uniques 1095; 2024-11-10T09:18:16,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T09:18:16,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b0abdf2a87854979a485fda53dc5aea8 2024-11-10T09:18:16,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7be5a083191f4f20a0a31a97fe74f5ac is 50, key is test_row_0/B:col10/1731230296023/Put/seqid=0 2024-11-10T09:18:16,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742311_1487 (size=12001) 2024-11-10T09:18:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:16,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731230296783, max timestamp=1731230296805Num uniques 1104; 2024-11-10T09:18:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230356801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:16,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-10T09:18:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:16,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731230296783, max timestamp=1731230296805Num uniques 1104; 2024-11-10T09:18:16,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-10T09:18:16,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731230296783, max timestamp=1731230296805Num uniques 1104; 2024-11-10T09:18:16,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-10T09:18:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:18:16,827 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:16,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T09:18:16,829 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:16,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:16,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:16,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731230296881, max timestamp=1731230296910Num uniques 1113; 2024-11-10T09:18:16,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-10T09:18:16,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731230296881, max timestamp=1731230296910Num uniques 1113; 2024-11-10T09:18:16,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731230296881, max timestamp=1731230296910Num uniques 1113; 2024-11-10T09:18:16,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-10T09:18:16,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-10T09:18:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:16,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731230296948, max timestamp=1731230296978Num uniques 1072; 2024-11-10T09:18:16,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-10T09:18:16,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731230296948, max timestamp=1731230296978Num uniques 1072; 2024-11-10T09:18:16,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731230296948, max timestamp=1731230296978Num uniques 1072; 2024-11-10T09:18:16,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-10T09:18:16,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-10T09:18:16,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:16,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731230297030, max timestamp=1731230297045Num uniques 1131; 2024-11-10T09:18:17,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-10T09:18:17,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731230297030, max timestamp=1731230297045Num uniques 1131; 2024-11-10T09:18:17,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731230297030, max timestamp=1731230297045Num uniques 1131; 2024-11-10T09:18:17,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-10T09:18:17,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-10T09:18:17,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731230297091, max timestamp=1731230297105Num uniques 1040; 2024-11-10T09:18:17,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-10T09:18:17,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731230297091, max timestamp=1731230297105Num uniques 1040; 2024-11-10T09:18:17,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731230297091, max timestamp=1731230297105Num uniques 1040; 2024-11-10T09:18:17,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-10T09:18:17,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-10T09:18:17,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731230297149, max timestamp=1731230297174Num uniques 1099; 2024-11-10T09:18:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T09:18:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731230297149, max timestamp=1731230297174Num uniques 1099; 2024-11-10T09:18:17,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T09:18:17,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731230297149, max timestamp=1731230297174Num uniques 1099; 2024-11-10T09:18:17,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T09:18:17,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7be5a083191f4f20a0a31a97fe74f5ac 2024-11-10T09:18:17,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/4d7abde2d72d427abfffa766e36d9c29 is 50, key is test_row_0/C:col10/1731230296023/Put/seqid=0 2024-11-10T09:18:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742312_1488 (size=12001) 2024-11-10T09:18:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230357265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731230297224, max timestamp=1731230297279Num uniques 1108; 2024-11-10T09:18:17,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T09:18:17,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731230297224, max timestamp=1731230297279Num uniques 1108; 2024-11-10T09:18:17,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731230297224, max timestamp=1731230297279Num uniques 1108; 2024-11-10T09:18:17,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T09:18:17,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T09:18:17,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731230297342, max timestamp=1731230297371Num uniques 1067; 2024-11-10T09:18:17,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-10T09:18:17,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731230297342, max timestamp=1731230297371Num uniques 1067; 2024-11-10T09:18:17,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-10T09:18:17,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731230297342, max timestamp=1731230297371Num uniques 1067; 2024-11-10T09:18:17,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-10T09:18:17,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731230297418, max timestamp=1731230297436Num uniques 1076; 2024-11-10T09:18:17,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731230297418, max timestamp=1731230297436Num uniques 1076; 2024-11-10T09:18:17,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731230297418, max timestamp=1731230297436Num uniques 1076; 2024-11-10T09:18:17,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-10T09:18:17,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-10T09:18:17,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-10T09:18:17,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:17,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731230297456, max timestamp=1731230297516Num uniques 1085; 2024-11-10T09:18:17,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-10T09:18:17,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731230297456, max timestamp=1731230297516Num uniques 1085; 2024-11-10T09:18:17,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731230297456, max timestamp=1731230297516Num uniques 1085; 2024-11-10T09:18:17,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-10T09:18:17,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-10T09:18:17,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230357585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731230297559, max timestamp=1731230297588Num uniques 1144; 2024-11-10T09:18:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-11-10T09:18:17,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731230297559, max timestamp=1731230297588Num uniques 1144; 2024-11-10T09:18:17,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-11-10T09:18:17,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731230297559, max timestamp=1731230297588Num uniques 1144; 2024-11-10T09:18:17,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:17,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-11-10T09:18:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/4d7abde2d72d427abfffa766e36d9c29 2024-11-10T09:18:17,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b0abdf2a87854979a485fda53dc5aea8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b0abdf2a87854979a485fda53dc5aea8 2024-11-10T09:18:17,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b0abdf2a87854979a485fda53dc5aea8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:18:17,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7be5a083191f4f20a0a31a97fe74f5ac as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7be5a083191f4f20a0a31a97fe74f5ac 2024-11-10T09:18:17,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7be5a083191f4f20a0a31a97fe74f5ac, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:18:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/4d7abde2d72d427abfffa766e36d9c29 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4d7abde2d72d427abfffa766e36d9c29 2024-11-10T09:18:17,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4d7abde2d72d427abfffa766e36d9c29, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:18:17,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=67.09 KB/68700 for 5b0c14639876e70a158c7f1b205d25b6 in 1261ms, sequenceid=117, compaction requested=true 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:17,628 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:17,628 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:17,630 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,630 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,630 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9492117dd4c849a6b280213a37021168, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/741b49eaf54f402ea663bb19db86e3a1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/1459f49f9208434b83196e4efdd1a2ae, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b0abdf2a87854979a485fda53dc5aea8] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=49.3 K 2024-11-10T09:18:17,630 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50a45725a4374aad8d5c44ced379089b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/323f574a9d284f35bbf94c5620f7cd90, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/a5fef21ac72348ac9a52d7582f6f7638, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7be5a083191f4f20a0a31a97fe74f5ac] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=47.0 K 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a45725a4374aad8d5c44ced379089b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230292773 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9492117dd4c849a6b280213a37021168, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230292773 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741b49eaf54f402ea663bb19db86e3a1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731230293508 2024-11-10T09:18:17,630 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 323f574a9d284f35bbf94c5620f7cd90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731230293508 2024-11-10T09:18:17,631 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1459f49f9208434b83196e4efdd1a2ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731230294715 2024-11-10T09:18:17,631 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting a5fef21ac72348ac9a52d7582f6f7638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731230294715 2024-11-10T09:18:17,631 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0abdf2a87854979a485fda53dc5aea8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230295970 2024-11-10T09:18:17,631 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be5a083191f4f20a0a31a97fe74f5ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230295970 2024-11-10T09:18:17,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=73.80 KB heapSize=193.03 KB 2024-11-10T09:18:17,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:17,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/44863c03fcf74350ba9b8bc1a4ab3d0b is 50, key is test_row_0/A:col10/1731230297629/Put/seqid=0 2024-11-10T09:18:17,638 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:17,639 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4eab64a6c3d84de6a52a4e8479ecf312 is 50, key is test_row_0/A:col10/1731230296023/Put/seqid=0 2024-11-10T09:18:17,642 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:17,642 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/1111f6685dc34db399a9dea5efcbd66c is 50, key is test_row_0/B:col10/1731230296023/Put/seqid=0 2024-11-10T09:18:17,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742313_1489 (size=14741) 2024-11-10T09:18:17,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/44863c03fcf74350ba9b8bc1a4ab3d0b 2024-11-10T09:18:17,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742314_1490 (size=12241) 2024-11-10T09:18:17,659 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4eab64a6c3d84de6a52a4e8479ecf312 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4eab64a6c3d84de6a52a4e8479ecf312 2024-11-10T09:18:17,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742315_1491 (size=12241) 2024-11-10T09:18:17,664 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into 4eab64a6c3d84de6a52a4e8479ecf312(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:17,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:17,664 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=12, startTime=1731230297628; duration=0sec 2024-11-10T09:18:17,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:17,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:17,664 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:17,666 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:17,666 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:17,666 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,666 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/e14cb508336943d4a18b9106bc105768, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f8f76a4116a74831b1ee7f8844777a5f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/55cc37953757435e9ddd9cfe2ad930f1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4d7abde2d72d427abfffa766e36d9c29] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=47.0 K 2024-11-10T09:18:17,666 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting e14cb508336943d4a18b9106bc105768, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731230292773 2024-11-10T09:18:17,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f76a4116a74831b1ee7f8844777a5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731230293508 2024-11-10T09:18:17,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55cc37953757435e9ddd9cfe2ad930f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731230294715 2024-11-10T09:18:17,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7abde2d72d427abfffa766e36d9c29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230295970 2024-11-10T09:18:17,667 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/1111f6685dc34db399a9dea5efcbd66c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1111f6685dc34db399a9dea5efcbd66c 2024-11-10T09:18:17,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/8e8064daed24455faadc88f7ba4fc551 is 50, key is test_row_0/B:col10/1731230297629/Put/seqid=0 2024-11-10T09:18:17,677 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into 1111f6685dc34db399a9dea5efcbd66c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:17,677 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:17,677 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=12, startTime=1731230297628; duration=0sec 2024-11-10T09:18:17,677 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:17,677 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:17,685 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:17,685 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/32f17da07c7b4e52abe290db0ba85b42 is 50, key is test_row_0/C:col10/1731230296023/Put/seqid=0 2024-11-10T09:18:17,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742316_1492 (size=12301) 2024-11-10T09:18:17,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742317_1493 (size=12241) 2024-11-10T09:18:17,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230357721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230357729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230357741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230357847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230357847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230357867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:17,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:17,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:18,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:18,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230358067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230358067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/8e8064daed24455faadc88f7ba4fc551 2024-11-10T09:18:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230358088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5bbdac4f9fa242b2b97ae94ba8996c5d is 50, key is test_row_0/C:col10/1731230297629/Put/seqid=0 2024-11-10T09:18:18,115 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/32f17da07c7b4e52abe290db0ba85b42 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/32f17da07c7b4e52abe290db0ba85b42 2024-11-10T09:18:18,120 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into 32f17da07c7b4e52abe290db0ba85b42(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:18,120 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:18,120 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=12, startTime=1731230297628; duration=0sec 2024-11-10T09:18:18,120 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:18,120 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742318_1494 (size=12301) 2024-11-10T09:18:18,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5bbdac4f9fa242b2b97ae94ba8996c5d 2024-11-10T09:18:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/44863c03fcf74350ba9b8bc1a4ab3d0b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/44863c03fcf74350ba9b8bc1a4ab3d0b 2024-11-10T09:18:18,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/44863c03fcf74350ba9b8bc1a4ab3d0b, entries=200, sequenceid=400, filesize=14.4 K 2024-11-10T09:18:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/8e8064daed24455faadc88f7ba4fc551 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/8e8064daed24455faadc88f7ba4fc551 2024-11-10T09:18:18,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/8e8064daed24455faadc88f7ba4fc551, entries=150, sequenceid=400, filesize=12.0 K 2024-11-10T09:18:18,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5bbdac4f9fa242b2b97ae94ba8996c5d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5bbdac4f9fa242b2b97ae94ba8996c5d 2024-11-10T09:18:18,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5bbdac4f9fa242b2b97ae94ba8996c5d, entries=150, sequenceid=400, filesize=12.0 K 2024-11-10T09:18:18,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~192.98 KB/197616, currentSize=107.34 KB/109920 for 5b0c14639876e70a158c7f1b205d25b6 in 507ms, sequenceid=400, compaction requested=false 2024-11-10T09:18:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:18,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:18:18,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T09:18:18,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T09:18:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:18,225 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T09:18:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/c397af195e7041a89f5d4c90b848e7cc is 50, key is test_row_0/A:col10/1731230297735/Put/seqid=0 2024-11-10T09:18:18,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742319_1495 (size=12301) 2024-11-10T09:18:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:18,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230358416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230358418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230358419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230358537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230358537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230358537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,647 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/c397af195e7041a89f5d4c90b848e7cc 2024-11-10T09:18:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/823a61a7f43741bab298749ae7010e71 is 50, key is test_row_0/B:col10/1731230297735/Put/seqid=0 2024-11-10T09:18:18,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742320_1496 (size=12301) 2024-11-10T09:18:18,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230358748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230358748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230358748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230358867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,872 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230358907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:18,911 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:19,058 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/823a61a7f43741bab298749ae7010e71 2024-11-10T09:18:19,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/a1be407704484d44964143cfa959bd6c is 50, key is test_row_0/C:col10/1731230297735/Put/seqid=0 2024-11-10T09:18:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742321_1497 (size=12301) 2024-11-10T09:18:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230359067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230359067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230359067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,467 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/a1be407704484d44964143cfa959bd6c 2024-11-10T09:18:19,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/c397af195e7041a89f5d4c90b848e7cc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/c397af195e7041a89f5d4c90b848e7cc 2024-11-10T09:18:19,475 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/c397af195e7041a89f5d4c90b848e7cc, entries=150, sequenceid=422, filesize=12.0 K 2024-11-10T09:18:19,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/823a61a7f43741bab298749ae7010e71 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/823a61a7f43741bab298749ae7010e71 2024-11-10T09:18:19,479 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/823a61a7f43741bab298749ae7010e71, entries=150, sequenceid=422, filesize=12.0 K 2024-11-10T09:18:19,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/a1be407704484d44964143cfa959bd6c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/a1be407704484d44964143cfa959bd6c 2024-11-10T09:18:19,484 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/a1be407704484d44964143cfa959bd6c, entries=150, sequenceid=422, filesize=12.0 K 2024-11-10T09:18:19,485 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5b0c14639876e70a158c7f1b205d25b6 in 1260ms, sequenceid=422, compaction requested=true 2024-11-10T09:18:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T09:18:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T09:18:19,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T09:18:19,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6570 sec 2024-11-10T09:18:19,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6630 sec 2024-11-10T09:18:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:19,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:18:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:19,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:19,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:19,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:19,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/3866056979dd4f8ca13428788d3d3a94 is 50, key is test_row_0/A:col10/1731230298406/Put/seqid=0 2024-11-10T09:18:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742322_1498 (size=14741) 2024-11-10T09:18:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230359722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230359722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230359722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230359847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230359848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230359848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/3866056979dd4f8ca13428788d3d3a94 2024-11-10T09:18:20,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/10aed50b418d45deb5d1e8cf1b750a11 is 50, key is test_row_0/B:col10/1731230298406/Put/seqid=0 2024-11-10T09:18:20,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742323_1499 (size=12301) 2024-11-10T09:18:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230360057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230360057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230360057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230360377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230360377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230360377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/10aed50b418d45deb5d1e8cf1b750a11 2024-11-10T09:18:20,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/546fec6311474634bab3c0b3e889b6dc is 50, key is test_row_0/C:col10/1731230298406/Put/seqid=0 2024-11-10T09:18:20,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742324_1500 (size=12301) 2024-11-10T09:18:20,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/546fec6311474634bab3c0b3e889b6dc 2024-11-10T09:18:20,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/3866056979dd4f8ca13428788d3d3a94 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/3866056979dd4f8ca13428788d3d3a94 2024-11-10T09:18:20,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/3866056979dd4f8ca13428788d3d3a94, entries=200, sequenceid=436, filesize=14.4 K 2024-11-10T09:18:20,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/10aed50b418d45deb5d1e8cf1b750a11 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/10aed50b418d45deb5d1e8cf1b750a11 2024-11-10T09:18:20,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/10aed50b418d45deb5d1e8cf1b750a11, entries=150, sequenceid=436, filesize=12.0 K 2024-11-10T09:18:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/546fec6311474634bab3c0b3e889b6dc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/546fec6311474634bab3c0b3e889b6dc 2024-11-10T09:18:20,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/546fec6311474634bab3c0b3e889b6dc, entries=150, sequenceid=436, filesize=12.0 K 2024-11-10T09:18:20,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 5b0c14639876e70a158c7f1b205d25b6 in 1256ms, sequenceid=436, compaction requested=true 2024-11-10T09:18:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:20,850 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:20,850 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:20,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:20,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:20,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:20,851 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:20,851 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:20,851 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:20,851 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:20,851 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:20,852 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4eab64a6c3d84de6a52a4e8479ecf312, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/44863c03fcf74350ba9b8bc1a4ab3d0b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/c397af195e7041a89f5d4c90b848e7cc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/3866056979dd4f8ca13428788d3d3a94] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=52.8 K 2024-11-10T09:18:20,852 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:20,852 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1111f6685dc34db399a9dea5efcbd66c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/8e8064daed24455faadc88f7ba4fc551, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/823a61a7f43741bab298749ae7010e71, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/10aed50b418d45deb5d1e8cf1b750a11] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=48.0 K 2024-11-10T09:18:20,852 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eab64a6c3d84de6a52a4e8479ecf312, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230295970 2024-11-10T09:18:20,853 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1111f6685dc34db399a9dea5efcbd66c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230295970 2024-11-10T09:18:20,853 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44863c03fcf74350ba9b8bc1a4ab3d0b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731230297588 2024-11-10T09:18:20,853 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8064daed24455faadc88f7ba4fc551, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731230297588 2024-11-10T09:18:20,853 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c397af195e7041a89f5d4c90b848e7cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731230297720 2024-11-10T09:18:20,854 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 823a61a7f43741bab298749ae7010e71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731230297720 2024-11-10T09:18:20,854 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 10aed50b418d45deb5d1e8cf1b750a11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731230298406 2024-11-10T09:18:20,854 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3866056979dd4f8ca13428788d3d3a94, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731230298406 2024-11-10T09:18:20,868 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#420 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:20,869 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b78fc1c742c5488abcaa39620dc0b7a7 is 50, key is test_row_0/A:col10/1731230298406/Put/seqid=0 2024-11-10T09:18:20,872 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:20,873 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/0cf17f442a80497791dda2e7c4a733bc is 50, key is test_row_0/B:col10/1731230298406/Put/seqid=0 2024-11-10T09:18:20,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742325_1501 (size=12677) 2024-11-10T09:18:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:20,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T09:18:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:20,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:20,913 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b78fc1c742c5488abcaa39620dc0b7a7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b78fc1c742c5488abcaa39620dc0b7a7 2024-11-10T09:18:20,917 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into b78fc1c742c5488abcaa39620dc0b7a7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:20,917 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:20,917 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=12, startTime=1731230300850; duration=0sec 2024-11-10T09:18:20,918 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:20,918 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:20,918 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:20,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:20,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:20,920 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:20,920 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/32f17da07c7b4e52abe290db0ba85b42, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5bbdac4f9fa242b2b97ae94ba8996c5d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/a1be407704484d44964143cfa959bd6c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/546fec6311474634bab3c0b3e889b6dc] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=48.0 K 2024-11-10T09:18:20,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f17da07c7b4e52abe290db0ba85b42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230295970 2024-11-10T09:18:20,920 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bbdac4f9fa242b2b97ae94ba8996c5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731230297588 2024-11-10T09:18:20,921 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1be407704484d44964143cfa959bd6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731230297720 2024-11-10T09:18:20,921 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546fec6311474634bab3c0b3e889b6dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731230298406 2024-11-10T09:18:20,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/d10221d31360494aaa55016751e510ed is 50, key is test_row_0/A:col10/1731230299678/Put/seqid=0 2024-11-10T09:18:20,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742326_1502 (size=12677) 2024-11-10T09:18:20,941 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:20,942 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/2454c8be228e4c9db63eb18881a76320 is 50, key is test_row_0/C:col10/1731230298406/Put/seqid=0 2024-11-10T09:18:20,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230360938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230360939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230360955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742327_1503 (size=14741) 2024-11-10T09:18:20,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742328_1504 (size=12677) 2024-11-10T09:18:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T09:18:20,998 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:20,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-10T09:18:21,001 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:18:21,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:21,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:21,016 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/2454c8be228e4c9db63eb18881a76320 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2454c8be228e4c9db63eb18881a76320 2024-11-10T09:18:21,023 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into 2454c8be228e4c9db63eb18881a76320(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:21,023 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:21,023 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=12, startTime=1731230300851; duration=0sec 2024-11-10T09:18:21,023 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:21,023 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:21,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230361058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230361067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230361077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:18:21,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:18:21,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:21,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:21,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:21,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:21,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:21,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230361287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230361287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230361297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:18:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:21,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:18:21,330 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/0cf17f442a80497791dda2e7c4a733bc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/0cf17f442a80497791dda2e7c4a733bc 2024-11-10T09:18:21,335 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into 0cf17f442a80497791dda2e7c4a733bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:21,335 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:21,335 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=12, startTime=1731230300850; duration=0sec 2024-11-10T09:18:21,335 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:21,335 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:21,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/d10221d31360494aaa55016751e510ed 2024-11-10T09:18:21,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/145379b8638d4097a8148f5ed015c352 is 50, key is test_row_0/B:col10/1731230299678/Put/seqid=0 2024-11-10T09:18:21,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742329_1505 (size=12301) 2024-11-10T09:18:21,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/145379b8638d4097a8148f5ed015c352 2024-11-10T09:18:21,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/78bf253810e34bc2aae33e7bc5a8a631 is 50, key is test_row_0/C:col10/1731230299678/Put/seqid=0 2024-11-10T09:18:21,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742330_1506 (size=12301) 2024-11-10T09:18:21,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/78bf253810e34bc2aae33e7bc5a8a631 2024-11-10T09:18:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/d10221d31360494aaa55016751e510ed as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d10221d31360494aaa55016751e510ed 2024-11-10T09:18:21,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d10221d31360494aaa55016751e510ed, entries=200, sequenceid=457, filesize=14.4 K 2024-11-10T09:18:21,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/145379b8638d4097a8148f5ed015c352 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/145379b8638d4097a8148f5ed015c352 2024-11-10T09:18:21,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/145379b8638d4097a8148f5ed015c352, entries=150, sequenceid=457, filesize=12.0 K 2024-11-10T09:18:21,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/78bf253810e34bc2aae33e7bc5a8a631 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/78bf253810e34bc2aae33e7bc5a8a631 2024-11-10T09:18:21,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/78bf253810e34bc2aae33e7bc5a8a631, entries=150, sequenceid=457, filesize=12.0 K 2024-11-10T09:18:21,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5b0c14639876e70a158c7f1b205d25b6 in 514ms, sequenceid=457, compaction requested=false 2024-11-10T09:18:21,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:21,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:18:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:21,462 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/34ab4f385aa445e48a3c3b0dd1333010 is 50, key is test_row_0/A:col10/1731230300952/Put/seqid=0 2024-11-10T09:18:21,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742331_1507 (size=12301) 2024-11-10T09:18:21,494 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/34ab4f385aa445e48a3c3b0dd1333010 2024-11-10T09:18:21,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/ece165a51d534f07b19fc0cb4ef30ab1 is 50, key is test_row_0/B:col10/1731230300952/Put/seqid=0 2024-11-10T09:18:21,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742332_1508 (size=12301) 2024-11-10T09:18:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:18:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230361662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230361671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230361672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230361787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230361787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230361797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:21,922 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/ece165a51d534f07b19fc0cb4ef30ab1 2024-11-10T09:18:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/f434bfeb9488417882c43a552d7ab4a7 is 50, key is test_row_0/C:col10/1731230300952/Put/seqid=0 2024-11-10T09:18:21,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742333_1509 (size=12301) 2024-11-10T09:18:21,938 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/f434bfeb9488417882c43a552d7ab4a7 2024-11-10T09:18:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/34ab4f385aa445e48a3c3b0dd1333010 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/34ab4f385aa445e48a3c3b0dd1333010 2024-11-10T09:18:21,956 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/34ab4f385aa445e48a3c3b0dd1333010, entries=150, sequenceid=471, filesize=12.0 K 2024-11-10T09:18:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/ece165a51d534f07b19fc0cb4ef30ab1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ece165a51d534f07b19fc0cb4ef30ab1 2024-11-10T09:18:21,959 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ece165a51d534f07b19fc0cb4ef30ab1, entries=150, sequenceid=471, filesize=12.0 K 2024-11-10T09:18:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/f434bfeb9488417882c43a552d7ab4a7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f434bfeb9488417882c43a552d7ab4a7 2024-11-10T09:18:21,963 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f434bfeb9488417882c43a552d7ab4a7, entries=150, sequenceid=471, filesize=12.0 K 2024-11-10T09:18:21,963 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5b0c14639876e70a158c7f1b205d25b6 in 501ms, sequenceid=471, compaction requested=true 2024-11-10T09:18:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:21,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-10T09:18:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-10T09:18:21,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T09:18:21,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 965 msec 2024-11-10T09:18:21,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 972 msec 2024-11-10T09:18:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:22,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-10T09:18:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:22,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/92148bb348ee46178ca4be966aab2136 is 50, key is test_row_0/A:col10/1731230301657/Put/seqid=0 2024-11-10T09:18:22,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742334_1510 (size=14741) 2024-11-10T09:18:22,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/92148bb348ee46178ca4be966aab2136 2024-11-10T09:18:22,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/1267d23a685949a7a49fe004f828087f is 50, key is test_row_0/B:col10/1731230301657/Put/seqid=0 2024-11-10T09:18:22,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742335_1511 (size=12301) 2024-11-10T09:18:22,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230362028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230362031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230362031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:18:22,137 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:22,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T09:18:22,142 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:22,143 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:22,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:18:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230362147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230362147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230362147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:18:22,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:18:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:22,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230362367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230362367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230362367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/1267d23a685949a7a49fe004f828087f 2024-11-10T09:18:22,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/7ae3665d93974a4e954b089aba9c560d is 50, key is test_row_0/C:col10/1731230301657/Put/seqid=0 2024-11-10T09:18:22,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742336_1512 (size=12301) 2024-11-10T09:18:22,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/7ae3665d93974a4e954b089aba9c560d 2024-11-10T09:18:22,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:18:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/92148bb348ee46178ca4be966aab2136 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/92148bb348ee46178ca4be966aab2136 2024-11-10T09:18:22,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/92148bb348ee46178ca4be966aab2136, entries=200, sequenceid=494, filesize=14.4 K 2024-11-10T09:18:22,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/1267d23a685949a7a49fe004f828087f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1267d23a685949a7a49fe004f828087f 2024-11-10T09:18:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:18:22,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1267d23a685949a7a49fe004f828087f, entries=150, sequenceid=494, filesize=12.0 K 2024-11-10T09:18:22,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/7ae3665d93974a4e954b089aba9c560d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/7ae3665d93974a4e954b089aba9c560d 2024-11-10T09:18:22,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/7ae3665d93974a4e954b089aba9c560d, entries=150, sequenceid=494, filesize=12.0 K 2024-11-10T09:18:22,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for 5b0c14639876e70a158c7f1b205d25b6 in 459ms, sequenceid=494, compaction requested=true 2024-11-10T09:18:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:22,472 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:22,472 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:22,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:22,475 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:22,475 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:22,475 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,475 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/0cf17f442a80497791dda2e7c4a733bc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/145379b8638d4097a8148f5ed015c352, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ece165a51d534f07b19fc0cb4ef30ab1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1267d23a685949a7a49fe004f828087f] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=48.4 K 2024-11-10T09:18:22,475 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cf17f442a80497791dda2e7c4a733bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731230298406 2024-11-10T09:18:22,476 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 145379b8638d4097a8148f5ed015c352, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731230299678 2024-11-10T09:18:22,476 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting ece165a51d534f07b19fc0cb4ef30ab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731230300930 2024-11-10T09:18:22,476 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1267d23a685949a7a49fe004f828087f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731230301657 2024-11-10T09:18:22,477 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:22,477 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:22,477 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,477 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b78fc1c742c5488abcaa39620dc0b7a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d10221d31360494aaa55016751e510ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/34ab4f385aa445e48a3c3b0dd1333010, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/92148bb348ee46178ca4be966aab2136] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=53.2 K 2024-11-10T09:18:22,477 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78fc1c742c5488abcaa39620dc0b7a7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731230298406 2024-11-10T09:18:22,477 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10221d31360494aaa55016751e510ed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731230299649 2024-11-10T09:18:22,477 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34ab4f385aa445e48a3c3b0dd1333010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731230300930 2024-11-10T09:18:22,478 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92148bb348ee46178ca4be966aab2136, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731230301657 2024-11-10T09:18:22,482 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:22,482 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/f04de1083dc143b5b693a1954ee75689 is 50, key is test_row_0/B:col10/1731230301657/Put/seqid=0 2024-11-10T09:18:22,486 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:22,486 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ece408fd7fe746e7be94498242d42d9a is 50, key is test_row_0/A:col10/1731230301657/Put/seqid=0 2024-11-10T09:18:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742337_1513 (size=12813) 2024-11-10T09:18:22,495 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/f04de1083dc143b5b693a1954ee75689 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f04de1083dc143b5b693a1954ee75689 2024-11-10T09:18:22,500 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into f04de1083dc143b5b693a1954ee75689(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:22,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:22,500 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=12, startTime=1731230302472; duration=0sec 2024-11-10T09:18:22,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:22,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:22,500 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:22,504 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:22,504 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:22,504 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,504 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2454c8be228e4c9db63eb18881a76320, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/78bf253810e34bc2aae33e7bc5a8a631, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f434bfeb9488417882c43a552d7ab4a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/7ae3665d93974a4e954b089aba9c560d] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=48.4 K 2024-11-10T09:18:22,505 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 2454c8be228e4c9db63eb18881a76320, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731230298406 2024-11-10T09:18:22,505 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bf253810e34bc2aae33e7bc5a8a631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731230299678 2024-11-10T09:18:22,505 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f434bfeb9488417882c43a552d7ab4a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731230300930 2024-11-10T09:18:22,505 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae3665d93974a4e954b089aba9c560d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731230301657 2024-11-10T09:18:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742338_1514 (size=12813) 2024-11-10T09:18:22,513 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ece408fd7fe746e7be94498242d42d9a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ece408fd7fe746e7be94498242d42d9a 2024-11-10T09:18:22,516 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:22,517 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/d67037481609401890f1041c8a741668 is 50, key is test_row_0/C:col10/1731230301657/Put/seqid=0 2024-11-10T09:18:22,519 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into ece408fd7fe746e7be94498242d42d9a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:22,519 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:22,519 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=12, startTime=1731230302472; duration=0sec 2024-11-10T09:18:22,519 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:22,519 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:22,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742339_1515 (size=12813) 2024-11-10T09:18:22,539 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/d67037481609401890f1041c8a741668 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/d67037481609401890f1041c8a741668 2024-11-10T09:18:22,550 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into d67037481609401890f1041c8a741668(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:22,550 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:22,550 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=12, startTime=1731230302472; duration=0sec 2024-11-10T09:18:22,550 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:22,550 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:22,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:18:22,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:22,608 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T09:18:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:22,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:22,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ff46c5490f084283a40f56f795b4e34e is 50, key is test_row_0/A:col10/1731230302029/Put/seqid=0 2024-11-10T09:18:22,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742340_1516 (size=12301) 2024-11-10T09:18:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:22,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:22,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230302712, max timestamp=1731230302744Num uniques 859; 2024-11-10T09:18:22,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:18:22,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230302723, max timestamp=1731230302744Num uniques 859; 2024-11-10T09:18:22,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:22,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230302723, max timestamp=1731230302744Num uniques 859; 2024-11-10T09:18:22,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:18:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:18:22,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230362809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230362810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230362821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230362927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230362927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230362927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,020 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ff46c5490f084283a40f56f795b4e34e 2024-11-10T09:18:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7ea2812a746547989334faebf86a3dbc is 50, key is test_row_0/B:col10/1731230302029/Put/seqid=0 2024-11-10T09:18:23,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742341_1517 (size=12301) 2024-11-10T09:18:23,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230363147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:23,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230363147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230363147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:18:23,436 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7ea2812a746547989334faebf86a3dbc 2024-11-10T09:18:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/4a074b41d2ff4d4b8ce171946e392c52 is 50, key is test_row_0/C:col10/1731230302029/Put/seqid=0 2024-11-10T09:18:23,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742342_1518 (size=12301) 2024-11-10T09:18:23,474 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/4a074b41d2ff4d4b8ce171946e392c52 2024-11-10T09:18:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ff46c5490f084283a40f56f795b4e34e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ff46c5490f084283a40f56f795b4e34e 2024-11-10T09:18:23,480 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ff46c5490f084283a40f56f795b4e34e, entries=150, sequenceid=506, filesize=12.0 K 2024-11-10T09:18:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7ea2812a746547989334faebf86a3dbc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7ea2812a746547989334faebf86a3dbc 2024-11-10T09:18:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230363467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230363468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230363468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:23,501 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7ea2812a746547989334faebf86a3dbc, entries=150, sequenceid=506, filesize=12.0 K 2024-11-10T09:18:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/4a074b41d2ff4d4b8ce171946e392c52 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4a074b41d2ff4d4b8ce171946e392c52 2024-11-10T09:18:23,508 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4a074b41d2ff4d4b8ce171946e392c52, entries=150, sequenceid=506, filesize=12.0 K 2024-11-10T09:18:23,513 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.26 KB/141575 for 5b0c14639876e70a158c7f1b205d25b6 in 906ms, sequenceid=506, compaction requested=false 2024-11-10T09:18:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T09:18:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T09:18:23,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T09:18:23,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-10T09:18:23,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.3790 sec 2024-11-10T09:18:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:23,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=144.97 KB heapSize=414.85 KB 2024-11-10T09:18:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:18:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:18:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:23,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:18:24,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/910afaa75f02477194d5ac89bcc38d80 is 50, key is test_row_0/A:col10/1731230302807/Put/seqid=0 2024-11-10T09:18:24,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742343_1519 (size=14741) 2024-11-10T09:18:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230364009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230364010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230364010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230364117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230364117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230364117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:18:24,298 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:24,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-10T09:18:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:18:24,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:24,300 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:24,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230364337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230364337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230364337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:18:24,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/910afaa75f02477194d5ac89bcc38d80 2024-11-10T09:18:24,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/9b20064df8594738bbf8f9fbf6a43c50 is 50, key is test_row_0/B:col10/1731230302807/Put/seqid=0 2024-11-10T09:18:24,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742344_1520 (size=12301) 2024-11-10T09:18:24,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/9b20064df8594738bbf8f9fbf6a43c50 2024-11-10T09:18:24,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/96dea96b8648414da75dcbe6cc29d2c9 is 50, key is test_row_0/C:col10/1731230302807/Put/seqid=0 2024-11-10T09:18:24,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742345_1521 (size=12301) 2024-11-10T09:18:24,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/96dea96b8648414da75dcbe6cc29d2c9 2024-11-10T09:18:24,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/910afaa75f02477194d5ac89bcc38d80 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/910afaa75f02477194d5ac89bcc38d80 2024-11-10T09:18:24,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/910afaa75f02477194d5ac89bcc38d80, entries=200, sequenceid=544, filesize=14.4 K 2024-11-10T09:18:24,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/9b20064df8594738bbf8f9fbf6a43c50 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/9b20064df8594738bbf8f9fbf6a43c50 2024-11-10T09:18:24,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/9b20064df8594738bbf8f9fbf6a43c50, entries=150, sequenceid=544, filesize=12.0 K 2024-11-10T09:18:24,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/96dea96b8648414da75dcbe6cc29d2c9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/96dea96b8648414da75dcbe6cc29d2c9 2024-11-10T09:18:24,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/96dea96b8648414da75dcbe6cc29d2c9, entries=150, sequenceid=544, filesize=12.0 K 2024-11-10T09:18:24,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~144.97 KB/148445, heapSize ~414.80 KB/424760, currentSize=33.54 KB/34350 for 5b0c14639876e70a158c7f1b205d25b6 in 451ms, sequenceid=544, compaction requested=true 2024-11-10T09:18:24,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:24,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:24,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:24,447 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:24,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:24,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:24,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:24,448 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:24,449 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:24,449 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:24,449 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:24,449 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:24,449 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:24,449 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ece408fd7fe746e7be94498242d42d9a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ff46c5490f084283a40f56f795b4e34e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/910afaa75f02477194d5ac89bcc38d80] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=38.9 K 2024-11-10T09:18:24,449 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:24,449 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f04de1083dc143b5b693a1954ee75689, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7ea2812a746547989334faebf86a3dbc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/9b20064df8594738bbf8f9fbf6a43c50] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.5 K 2024-11-10T09:18:24,450 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece408fd7fe746e7be94498242d42d9a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731230301657 2024-11-10T09:18:24,450 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f04de1083dc143b5b693a1954ee75689, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731230301657 2024-11-10T09:18:24,450 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff46c5490f084283a40f56f795b4e34e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731230302013 2024-11-10T09:18:24,450 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ea2812a746547989334faebf86a3dbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731230302013 2024-11-10T09:18:24,450 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b20064df8594738bbf8f9fbf6a43c50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731230302807 2024-11-10T09:18:24,450 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 910afaa75f02477194d5ac89bcc38d80, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731230302807 2024-11-10T09:18:24,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:24,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:24,452 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:24,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/53dc12d29f084a1c9e79d015a20233f0 is 50, key is test_row_0/A:col10/1731230304008/Put/seqid=0 2024-11-10T09:18:24,470 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:24,470 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/f7b5f865a5dd4670b811adc99e0dd2c1 is 50, key is test_row_0/B:col10/1731230302807/Put/seqid=0 2024-11-10T09:18:24,474 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:24,475 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/40a6561e87b5497f8156c73ec18326b4 is 50, key is test_row_0/A:col10/1731230302807/Put/seqid=0 2024-11-10T09:18:24,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742346_1522 (size=12301) 2024-11-10T09:18:24,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742347_1523 (size=12915) 2024-11-10T09:18:24,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742348_1524 (size=12915) 2024-11-10T09:18:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:18:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230304684, max timestamp=1731230304718Num uniques 859; 2024-11-10T09:18:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230304684, max timestamp=1731230304718Num uniques 859; 2024-11-10T09:18:24,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:24,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230304684, max timestamp=1731230304718Num uniques 859; 2024-11-10T09:18:24,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:24,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230304759, max timestamp=1731230304778Num uniques 1068; 2024-11-10T09:18:24,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T09:18:24,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230304759, max timestamp=1731230304778Num uniques 1068; 2024-11-10T09:18:24,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T09:18:24,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230304759, max timestamp=1731230304778Num uniques 1068; 2024-11-10T09:18:24,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T09:18:24,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230364834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230364836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230304827, max timestamp=1731230304848Num uniques 1077; 2024-11-10T09:18:24,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T09:18:24,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230304827, max timestamp=1731230304848Num uniques 1077; 2024-11-10T09:18:24,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T09:18:24,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230304827, max timestamp=1731230304848Num uniques 1077; 2024-11-10T09:18:24,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T09:18:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:24,892 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/53dc12d29f084a1c9e79d015a20233f0 2024-11-10T09:18:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/382cdee5797a408d8c8bf7bf53cb4a1f is 50, key is test_row_0/B:col10/1731230304008/Put/seqid=0 2024-11-10T09:18:24,910 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/f7b5f865a5dd4670b811adc99e0dd2c1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f7b5f865a5dd4670b811adc99e0dd2c1 2024-11-10T09:18:24,914 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into f7b5f865a5dd4670b811adc99e0dd2c1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:24,914 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:24,914 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=13, startTime=1731230304448; duration=0sec 2024-11-10T09:18:24,915 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:24,915 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:24,915 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:24,918 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:24,918 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:24,918 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:24,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742349_1525 (size=12301) 2024-11-10T09:18:24,918 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/d67037481609401890f1041c8a741668, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4a074b41d2ff4d4b8ce171946e392c52, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/96dea96b8648414da75dcbe6cc29d2c9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.5 K 2024-11-10T09:18:24,919 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d67037481609401890f1041c8a741668, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731230301657 2024-11-10T09:18:24,919 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/382cdee5797a408d8c8bf7bf53cb4a1f 2024-11-10T09:18:24,919 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a074b41d2ff4d4b8ce171946e392c52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731230302013 2024-11-10T09:18:24,919 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 96dea96b8648414da75dcbe6cc29d2c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731230302807 2024-11-10T09:18:24,921 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/40a6561e87b5497f8156c73ec18326b4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/40a6561e87b5497f8156c73ec18326b4 2024-11-10T09:18:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/c73b1cd5f61c4111a5bd3b190340b5f1 is 50, key is test_row_0/C:col10/1731230304008/Put/seqid=0 2024-11-10T09:18:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:18:24,928 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into 40a6561e87b5497f8156c73ec18326b4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:24,928 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:24,929 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=13, startTime=1731230304447; duration=0sec 2024-11-10T09:18:24,929 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:24,929 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:24,931 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:24,932 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5eff7e628de44c35abdc2e1892fa1f55 is 50, key is test_row_0/C:col10/1731230302807/Put/seqid=0 2024-11-10T09:18:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742350_1526 (size=12301) 2024-11-10T09:18:24,941 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/c73b1cd5f61c4111a5bd3b190340b5f1 2024-11-10T09:18:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/53dc12d29f084a1c9e79d015a20233f0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/53dc12d29f084a1c9e79d015a20233f0 2024-11-10T09:18:24,951 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/53dc12d29f084a1c9e79d015a20233f0, entries=150, sequenceid=552, filesize=12.0 K 2024-11-10T09:18:24,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/382cdee5797a408d8c8bf7bf53cb4a1f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/382cdee5797a408d8c8bf7bf53cb4a1f 2024-11-10T09:18:24,958 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/382cdee5797a408d8c8bf7bf53cb4a1f, entries=150, sequenceid=552, filesize=12.0 K 2024-11-10T09:18:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742351_1527 (size=12915) 2024-11-10T09:18:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/c73b1cd5f61c4111a5bd3b190340b5f1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c73b1cd5f61c4111a5bd3b190340b5f1 2024-11-10T09:18:24,971 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c73b1cd5f61c4111a5bd3b190340b5f1, entries=150, sequenceid=552, filesize=12.0 K 2024-11-10T09:18:24,972 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=100.63 KB/103050 for 5b0c14639876e70a158c7f1b205d25b6 in 520ms, sequenceid=552, compaction requested=false 2024-11-10T09:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-10T09:18:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-10T09:18:24,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=107.34 KB heapSize=303.61 KB 2024-11-10T09:18:24,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T09:18:24,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 674 msec 2024-11-10T09:18:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:24,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 679 msec 2024-11-10T09:18:24,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/98f61f2c2d06463c9c507a74c2bd0d30 is 50, key is test_row_0/A:col10/1731230304968/Put/seqid=0 2024-11-10T09:18:24,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742352_1528 (size=17181) 2024-11-10T09:18:25,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230365015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230365030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230365035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230365138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230365147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230365157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230365347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,365 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5eff7e628de44c35abdc2e1892fa1f55 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5eff7e628de44c35abdc2e1892fa1f55 2024-11-10T09:18:25,370 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into 5eff7e628de44c35abdc2e1892fa1f55(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:25,370 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:25,370 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=13, startTime=1731230304448; duration=0sec 2024-11-10T09:18:25,370 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:25,370 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:25,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230365367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230365378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/98f61f2c2d06463c9c507a74c2bd0d30 2024-11-10T09:18:25,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/5efedd7cf8944d0cbb49a315f883d988 is 50, key is test_row_0/B:col10/1731230304968/Put/seqid=0 2024-11-10T09:18:25,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742353_1529 (size=12301) 2024-11-10T09:18:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:18:25,437 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:25,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T09:18:25,440 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:18:25,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:25,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:18:25,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T09:18:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230365669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230365677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230365697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:25,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T09:18:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:18:25,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/5efedd7cf8944d0cbb49a315f883d988 2024-11-10T09:18:25,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5c0f3740c0984100a2d641b365998eb3 is 50, key is test_row_0/C:col10/1731230304968/Put/seqid=0 2024-11-10T09:18:25,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742354_1530 (size=12301) 2024-11-10T09:18:25,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5c0f3740c0984100a2d641b365998eb3 2024-11-10T09:18:25,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/98f61f2c2d06463c9c507a74c2bd0d30 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/98f61f2c2d06463c9c507a74c2bd0d30 2024-11-10T09:18:25,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/98f61f2c2d06463c9c507a74c2bd0d30, entries=250, sequenceid=621, filesize=16.8 K 2024-11-10T09:18:25,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/5efedd7cf8944d0cbb49a315f883d988 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5efedd7cf8944d0cbb49a315f883d988 2024-11-10T09:18:25,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5efedd7cf8944d0cbb49a315f883d988, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T09:18:25,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5c0f3740c0984100a2d641b365998eb3 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5c0f3740c0984100a2d641b365998eb3 2024-11-10T09:18:25,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5c0f3740c0984100a2d641b365998eb3, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T09:18:25,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~303.56 KB/310848, currentSize=73.80 KB/75570 for 5b0c14639876e70a158c7f1b205d25b6 in 854ms, sequenceid=621, compaction requested=true 2024-11-10T09:18:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:25,830 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:25,830 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:25,830 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:25,830 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:25,830 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:25,830 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:25,830 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,830 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,831 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f7b5f865a5dd4670b811adc99e0dd2c1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/382cdee5797a408d8c8bf7bf53cb4a1f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5efedd7cf8944d0cbb49a315f883d988] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.6 K 2024-11-10T09:18:25,831 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/40a6561e87b5497f8156c73ec18326b4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/53dc12d29f084a1c9e79d015a20233f0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/98f61f2c2d06463c9c507a74c2bd0d30] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=41.4 K 2024-11-10T09:18:25,831 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b5f865a5dd4670b811adc99e0dd2c1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731230302807 2024-11-10T09:18:25,831 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a6561e87b5497f8156c73ec18326b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731230302807 2024-11-10T09:18:25,831 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53dc12d29f084a1c9e79d015a20233f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731230304008 2024-11-10T09:18:25,831 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 382cdee5797a408d8c8bf7bf53cb4a1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731230304008 2024-11-10T09:18:25,831 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f61f2c2d06463c9c507a74c2bd0d30, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731230304827 2024-11-10T09:18:25,831 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5efedd7cf8944d0cbb49a315f883d988, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731230304827 2024-11-10T09:18:25,838 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:25,839 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/f11927768f3a4582b815c9eb1651e02e is 50, key is test_row_0/B:col10/1731230304968/Put/seqid=0 2024-11-10T09:18:25,842 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:25,843 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/852f858a09494f12b6b4963cd08c31bb is 50, key is test_row_0/A:col10/1731230304968/Put/seqid=0 2024-11-10T09:18:25,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742355_1531 (size=13017) 2024-11-10T09:18:25,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742356_1532 (size=13017) 2024-11-10T09:18:25,859 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/f11927768f3a4582b815c9eb1651e02e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f11927768f3a4582b815c9eb1651e02e 2024-11-10T09:18:25,861 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/852f858a09494f12b6b4963cd08c31bb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/852f858a09494f12b6b4963cd08c31bb 2024-11-10T09:18:25,868 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into 852f858a09494f12b6b4963cd08c31bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:25,868 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:25,868 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=13, startTime=1731230305830; duration=0sec 2024-11-10T09:18:25,868 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:25,868 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:25,868 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:25,869 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into f11927768f3a4582b815c9eb1651e02e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:25,869 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:25,869 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=13, startTime=1731230305830; duration=0sec 2024-11-10T09:18:25,869 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:25,869 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:25,869 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:25,869 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:25,869 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,869 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5eff7e628de44c35abdc2e1892fa1f55, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c73b1cd5f61c4111a5bd3b190340b5f1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5c0f3740c0984100a2d641b365998eb3] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.6 K 2024-11-10T09:18:25,869 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eff7e628de44c35abdc2e1892fa1f55, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731230302807 2024-11-10T09:18:25,870 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c73b1cd5f61c4111a5bd3b190340b5f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731230304008 2024-11-10T09:18:25,870 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0f3740c0984100a2d641b365998eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731230304827 2024-11-10T09:18:25,877 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:25,877 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/c8a0b1fb61fb4163bce9f6750655a126 is 50, key is test_row_0/C:col10/1731230304968/Put/seqid=0 2024-11-10T09:18:25,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742357_1533 (size=13017) 2024-11-10T09:18:25,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:25,900 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:25,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/51d14c8207a345ed8ac68e753e32af3f is 50, key is test_row_0/A:col10/1731230305001/Put/seqid=0 2024-11-10T09:18:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742358_1534 (size=12301) 2024-11-10T09:18:25,915 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/51d14c8207a345ed8ac68e753e32af3f 2024-11-10T09:18:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/c65eb310bd824a059632c44e436f72ac is 50, key is test_row_0/B:col10/1731230305001/Put/seqid=0 2024-11-10T09:18:25,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742359_1535 (size=12301) 2024-11-10T09:18:25,939 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/c65eb310bd824a059632c44e436f72ac 2024-11-10T09:18:25,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5215de822d38475db7fc9c2609214c45 is 50, key is test_row_0/C:col10/1731230305001/Put/seqid=0 2024-11-10T09:18:25,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742360_1536 (size=12301) 2024-11-10T09:18:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:18:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:26,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230366248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230366250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230366251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,287 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/c8a0b1fb61fb4163bce9f6750655a126 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c8a0b1fb61fb4163bce9f6750655a126 2024-11-10T09:18:26,291 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into c8a0b1fb61fb4163bce9f6750655a126(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:26,291 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:26,291 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=13, startTime=1731230305830; duration=0sec 2024-11-10T09:18:26,291 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:26,291 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:26,352 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5215de822d38475db7fc9c2609214c45 2024-11-10T09:18:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/51d14c8207a345ed8ac68e753e32af3f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/51d14c8207a345ed8ac68e753e32af3f 2024-11-10T09:18:26,364 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/51d14c8207a345ed8ac68e753e32af3f, entries=150, sequenceid=638, filesize=12.0 K 2024-11-10T09:18:26,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/c65eb310bd824a059632c44e436f72ac as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c65eb310bd824a059632c44e436f72ac 2024-11-10T09:18:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230366357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230366367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,377 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c65eb310bd824a059632c44e436f72ac, entries=150, sequenceid=638, filesize=12.0 K 2024-11-10T09:18:26,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/5215de822d38475db7fc9c2609214c45 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5215de822d38475db7fc9c2609214c45 2024-11-10T09:18:26,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230366372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,384 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5215de822d38475db7fc9c2609214c45, entries=150, sequenceid=638, filesize=12.0 K 2024-11-10T09:18:26,385 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5b0c14639876e70a158c7f1b205d25b6 in 485ms, sequenceid=638, compaction requested=false 2024-11-10T09:18:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:26,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T09:18:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T09:18:26,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T09:18:26,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 945 msec 2024-11-10T09:18:26,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 954 msec 2024-11-10T09:18:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:18:26,578 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:26,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-10T09:18:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:18:26,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:26,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:26,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:26,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T09:18:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:26,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ba35a2bf69364dd68d39c4b8b1f48a13 is 50, key is test_row_0/A:col10/1731230306246/Put/seqid=0 2024-11-10T09:18:26,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742361_1537 (size=14741) 2024-11-10T09:18:26,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ba35a2bf69364dd68d39c4b8b1f48a13 2024-11-10T09:18:26,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/73ecd526600b4984a876ca4687455ffd is 50, key is test_row_0/B:col10/1731230306246/Put/seqid=0 2024-11-10T09:18:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742362_1538 (size=12301) 2024-11-10T09:18:26,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230366611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230366612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230366620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:18:26,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:18:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:26,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:26,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230366727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230366737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230366737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:18:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:26,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:18:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230366947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230366948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230366948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/73ecd526600b4984a876ca4687455ffd 2024-11-10T09:18:27,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/758b0451fa2a478ab951d299e3f33b51 is 50, key is test_row_0/C:col10/1731230306246/Put/seqid=0 2024-11-10T09:18:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742363_1539 (size=12301) 2024-11-10T09:18:27,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:18:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:18:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230367257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230367269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230367269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:18:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:27,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/758b0451fa2a478ab951d299e3f33b51 2024-11-10T09:18:27,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/ba35a2bf69364dd68d39c4b8b1f48a13 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ba35a2bf69364dd68d39c4b8b1f48a13 2024-11-10T09:18:27,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ba35a2bf69364dd68d39c4b8b1f48a13, entries=200, sequenceid=658, filesize=14.4 K 2024-11-10T09:18:27,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/73ecd526600b4984a876ca4687455ffd as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/73ecd526600b4984a876ca4687455ffd 2024-11-10T09:18:27,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/73ecd526600b4984a876ca4687455ffd, entries=150, sequenceid=658, filesize=12.0 K 2024-11-10T09:18:27,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/758b0451fa2a478ab951d299e3f33b51 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/758b0451fa2a478ab951d299e3f33b51 2024-11-10T09:18:27,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/758b0451fa2a478ab951d299e3f33b51, entries=150, sequenceid=658, filesize=12.0 K 2024-11-10T09:18:27,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5b0c14639876e70a158c7f1b205d25b6 in 850ms, sequenceid=658, compaction requested=true 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:27,433 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:27,433 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:27,435 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:27,435 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:27,435 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,435 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f11927768f3a4582b815c9eb1651e02e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c65eb310bd824a059632c44e436f72ac, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/73ecd526600b4984a876ca4687455ffd] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.7 K 2024-11-10T09:18:27,435 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:27,435 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:27,435 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,435 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/852f858a09494f12b6b4963cd08c31bb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/51d14c8207a345ed8ac68e753e32af3f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ba35a2bf69364dd68d39c4b8b1f48a13] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=39.1 K 2024-11-10T09:18:27,435 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 852f858a09494f12b6b4963cd08c31bb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731230304827 2024-11-10T09:18:27,435 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f11927768f3a4582b815c9eb1651e02e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731230304827 2024-11-10T09:18:27,436 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d14c8207a345ed8ac68e753e32af3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731230305001 2024-11-10T09:18:27,436 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c65eb310bd824a059632c44e436f72ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731230305001 2024-11-10T09:18:27,437 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ecd526600b4984a876ca4687455ffd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731230306246 2024-11-10T09:18:27,438 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba35a2bf69364dd68d39c4b8b1f48a13, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731230306246 2024-11-10T09:18:27,444 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:27,445 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/c1d36a84ffbd4403b1881e87233887c4 is 50, key is test_row_0/B:col10/1731230306246/Put/seqid=0 2024-11-10T09:18:27,446 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:27,446 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/2da46de98b0e4bbbb54b745229c1f0b9 is 50, key is test_row_0/A:col10/1731230306246/Put/seqid=0 2024-11-10T09:18:27,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742364_1540 (size=13119) 2024-11-10T09:18:27,462 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/2da46de98b0e4bbbb54b745229c1f0b9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/2da46de98b0e4bbbb54b745229c1f0b9 2024-11-10T09:18:27,478 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into 2da46de98b0e4bbbb54b745229c1f0b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:27,479 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:27,479 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=13, startTime=1731230307433; duration=0sec 2024-11-10T09:18:27,479 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:27,479 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:27,479 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:27,479 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:27,480 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:27,480 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,480 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c8a0b1fb61fb4163bce9f6750655a126, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5215de822d38475db7fc9c2609214c45, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/758b0451fa2a478ab951d299e3f33b51] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.7 K 2024-11-10T09:18:27,480 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8a0b1fb61fb4163bce9f6750655a126, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731230304827 2024-11-10T09:18:27,480 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5215de822d38475db7fc9c2609214c45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1731230305001 2024-11-10T09:18:27,480 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 758b0451fa2a478ab951d299e3f33b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731230306246 2024-11-10T09:18:27,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742365_1541 (size=13119) 2024-11-10T09:18:27,491 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:27,492 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/35e6ef5584204edd986beceb9c33feea is 50, key is test_row_0/C:col10/1731230306246/Put/seqid=0 2024-11-10T09:18:27,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742366_1542 (size=13119) 2024-11-10T09:18:27,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:27,504 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/d7149b31e4604c7592971b848d2f6ae1 is 50, key is test_row_0/A:col10/1731230306619/Put/seqid=0 2024-11-10T09:18:27,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742367_1543 (size=12301) 2024-11-10T09:18:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:18:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:27,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230367843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230367844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230367861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,895 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/c1d36a84ffbd4403b1881e87233887c4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c1d36a84ffbd4403b1881e87233887c4 2024-11-10T09:18:27,902 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into c1d36a84ffbd4403b1881e87233887c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:27,902 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:27,902 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=13, startTime=1731230307433; duration=0sec 2024-11-10T09:18:27,902 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:27,902 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:27,908 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/35e6ef5584204edd986beceb9c33feea as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/35e6ef5584204edd986beceb9c33feea 2024-11-10T09:18:27,911 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into 35e6ef5584204edd986beceb9c33feea(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:27,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:27,912 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=13, startTime=1731230307433; duration=0sec 2024-11-10T09:18:27,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:27,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:27,925 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/d7149b31e4604c7592971b848d2f6ae1 2024-11-10T09:18:27,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/115aa146c05648c9a59920ace619a67b is 50, key is test_row_0/B:col10/1731230306619/Put/seqid=0 2024-11-10T09:18:27,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742368_1544 (size=12301) 2024-11-10T09:18:27,967 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/115aa146c05648c9a59920ace619a67b 2024-11-10T09:18:27,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230367968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230367968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230367977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/2fce20df17274527b65d6d43d84c57e5 is 50, key is test_row_0/C:col10/1731230306619/Put/seqid=0 2024-11-10T09:18:28,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742369_1545 (size=12301) 2024-11-10T09:18:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230368187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230368187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230368197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,416 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/2fce20df17274527b65d6d43d84c57e5 2024-11-10T09:18:28,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/d7149b31e4604c7592971b848d2f6ae1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d7149b31e4604c7592971b848d2f6ae1 2024-11-10T09:18:28,423 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d7149b31e4604c7592971b848d2f6ae1, entries=150, sequenceid=672, filesize=12.0 K 2024-11-10T09:18:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/115aa146c05648c9a59920ace619a67b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/115aa146c05648c9a59920ace619a67b 2024-11-10T09:18:28,427 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/115aa146c05648c9a59920ace619a67b, entries=150, sequenceid=672, filesize=12.0 K 2024-11-10T09:18:28,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/2fce20df17274527b65d6d43d84c57e5 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2fce20df17274527b65d6d43d84c57e5 2024-11-10T09:18:28,433 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2fce20df17274527b65d6d43d84c57e5, entries=150, sequenceid=672, filesize=12.0 K 2024-11-10T09:18:28,434 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5b0c14639876e70a158c7f1b205d25b6 in 930ms, sequenceid=672, compaction requested=false 2024-11-10T09:18:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-10T09:18:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-10T09:18:28,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T09:18:28,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8560 sec 2024-11-10T09:18:28,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.8600 sec 2024-11-10T09:18:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:28,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:28,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/6c47bd9a667846c0be762a33b784a780 is 50, key is test_row_0/A:col10/1731230307840/Put/seqid=0 2024-11-10T09:18:28,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742370_1546 (size=14741) 2024-11-10T09:18:28,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230368530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230368539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230368539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230368648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230368648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230368648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:18:28,728 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:28,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-10T09:18:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:18:28,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:28,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:28,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:18:28,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230368857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230368857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230368857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T09:18:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:28,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:28,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/6c47bd9a667846c0be762a33b784a780 2024-11-10T09:18:28,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/460ccc3273f14e8fa42420d87e447ecf is 50, key is test_row_0/B:col10/1731230307840/Put/seqid=0 2024-11-10T09:18:28,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742371_1547 (size=12301) 2024-11-10T09:18:28,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/460ccc3273f14e8fa42420d87e447ecf 2024-11-10T09:18:28,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/73f4f042c1544054b1bf1d739f0481a9 is 50, key is test_row_0/C:col10/1731230307840/Put/seqid=0 2024-11-10T09:18:28,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1731230368928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,939 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742372_1548 (size=12301) 2024-11-10T09:18:28,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/73f4f042c1544054b1bf1d739f0481a9 2024-11-10T09:18:28,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/6c47bd9a667846c0be762a33b784a780 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/6c47bd9a667846c0be762a33b784a780 2024-11-10T09:18:28,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/6c47bd9a667846c0be762a33b784a780, entries=200, sequenceid=694, filesize=14.4 K 2024-11-10T09:18:28,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/460ccc3273f14e8fa42420d87e447ecf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/460ccc3273f14e8fa42420d87e447ecf 2024-11-10T09:18:28,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38444 deadline: 1731230368947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:28,953 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in 5b0c14639876e70a158c7f1b205d25b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18321 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:28,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/460ccc3273f14e8fa42420d87e447ecf, entries=150, sequenceid=694, filesize=12.0 K 2024-11-10T09:18:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/73f4f042c1544054b1bf1d739f0481a9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/73f4f042c1544054b1bf1d739f0481a9 2024-11-10T09:18:28,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/73f4f042c1544054b1bf1d739f0481a9, entries=150, sequenceid=694, filesize=12.0 K 2024-11-10T09:18:28,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5b0c14639876e70a158c7f1b205d25b6 in 450ms, sequenceid=694, compaction requested=true 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:28,962 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:28,962 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:28,964 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:28,964 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:28,964 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:28,964 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:28,964 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:28,964 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:28,964 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c1d36a84ffbd4403b1881e87233887c4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/115aa146c05648c9a59920ace619a67b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/460ccc3273f14e8fa42420d87e447ecf] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.8 K 2024-11-10T09:18:28,964 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/2da46de98b0e4bbbb54b745229c1f0b9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d7149b31e4604c7592971b848d2f6ae1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/6c47bd9a667846c0be762a33b784a780] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=39.2 K 2024-11-10T09:18:28,966 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d36a84ffbd4403b1881e87233887c4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731230306246 2024-11-10T09:18:28,966 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da46de98b0e4bbbb54b745229c1f0b9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731230306246 2024-11-10T09:18:28,966 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7149b31e4604c7592971b848d2f6ae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731230306609 2024-11-10T09:18:28,966 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 115aa146c05648c9a59920ace619a67b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731230306609 2024-11-10T09:18:28,966 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c47bd9a667846c0be762a33b784a780, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731230307840 2024-11-10T09:18:28,966 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 460ccc3273f14e8fa42420d87e447ecf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731230307840 2024-11-10T09:18:28,980 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:28,980 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4b1e8fb4754e48b7a6525ac41363864f is 50, key is test_row_0/A:col10/1731230307840/Put/seqid=0 2024-11-10T09:18:28,982 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:28,983 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/78dca70a8d524edaaadbf39e263d873d is 50, key is test_row_0/B:col10/1731230307840/Put/seqid=0 2024-11-10T09:18:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742374_1550 (size=13221) 2024-11-10T09:18:29,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742373_1549 (size=13221) 2024-11-10T09:18:29,020 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4b1e8fb4754e48b7a6525ac41363864f as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4b1e8fb4754e48b7a6525ac41363864f 2024-11-10T09:18:29,026 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into 4b1e8fb4754e48b7a6525ac41363864f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:29,026 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:29,026 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=13, startTime=1731230308962; duration=0sec 2024-11-10T09:18:29,026 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:29,026 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:29,026 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:29,027 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:29,027 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:29,027 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:29,027 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/35e6ef5584204edd986beceb9c33feea, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2fce20df17274527b65d6d43d84c57e5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/73f4f042c1544054b1bf1d739f0481a9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.8 K 2024-11-10T09:18:29,028 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e6ef5584204edd986beceb9c33feea, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731230306246 2024-11-10T09:18:29,028 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fce20df17274527b65d6d43d84c57e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731230306609 2024-11-10T09:18:29,028 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73f4f042c1544054b1bf1d739f0481a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731230307840 2024-11-10T09:18:29,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T09:18:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:29,036 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:18:29,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:29,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:29,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:29,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:29,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:29,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:29,040 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:29,040 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/929952d748f24e68b1e7533c2dd5e6a7 is 50, key is test_row_0/C:col10/1731230307840/Put/seqid=0 2024-11-10T09:18:29,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b525c3f07e3b4581a6cfafd784f0a44c is 50, key is test_row_0/A:col10/1731230308528/Put/seqid=0 2024-11-10T09:18:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:18:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742375_1551 (size=13221) 2024-11-10T09:18:29,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742376_1552 (size=12301) 2024-11-10T09:18:29,073 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=707 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b525c3f07e3b4581a6cfafd784f0a44c 2024-11-10T09:18:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/475d5ec271334c7997d4c7c6399cd822 is 50, key is test_row_0/B:col10/1731230308528/Put/seqid=0 2024-11-10T09:18:29,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742377_1553 (size=12301) 2024-11-10T09:18:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:29,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230369260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230369261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230369270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:18:29,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230369378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230369387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230369388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,414 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/78dca70a8d524edaaadbf39e263d873d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/78dca70a8d524edaaadbf39e263d873d 2024-11-10T09:18:29,417 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into 78dca70a8d524edaaadbf39e263d873d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:29,417 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:29,417 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=13, startTime=1731230308962; duration=0sec 2024-11-10T09:18:29,417 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:29,417 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:29,466 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/929952d748f24e68b1e7533c2dd5e6a7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/929952d748f24e68b1e7533c2dd5e6a7 2024-11-10T09:18:29,474 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into 929952d748f24e68b1e7533c2dd5e6a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:29,474 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:29,474 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=13, startTime=1731230308962; duration=0sec 2024-11-10T09:18:29,474 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:29,474 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:29,488 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=707 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/475d5ec271334c7997d4c7c6399cd822 2024-11-10T09:18:29,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/1fc5e2098795486da125c60edee6fda1 is 50, key is test_row_0/C:col10/1731230308528/Put/seqid=0 2024-11-10T09:18:29,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742378_1554 (size=12301) 2024-11-10T09:18:29,528 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=707 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/1fc5e2098795486da125c60edee6fda1 2024-11-10T09:18:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/b525c3f07e3b4581a6cfafd784f0a44c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b525c3f07e3b4581a6cfafd784f0a44c 2024-11-10T09:18:29,540 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b525c3f07e3b4581a6cfafd784f0a44c, entries=150, sequenceid=707, filesize=12.0 K 2024-11-10T09:18:29,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/475d5ec271334c7997d4c7c6399cd822 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/475d5ec271334c7997d4c7c6399cd822 2024-11-10T09:18:29,546 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/475d5ec271334c7997d4c7c6399cd822, entries=150, sequenceid=707, filesize=12.0 K 2024-11-10T09:18:29,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/1fc5e2098795486da125c60edee6fda1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/1fc5e2098795486da125c60edee6fda1 2024-11-10T09:18:29,553 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/1fc5e2098795486da125c60edee6fda1, entries=150, sequenceid=707, filesize=12.0 K 2024-11-10T09:18:29,554 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5b0c14639876e70a158c7f1b205d25b6 in 519ms, sequenceid=707, compaction requested=false 2024-11-10T09:18:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-10T09:18:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-10T09:18:29,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T09:18:29,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 827 msec 2024-11-10T09:18:29,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 831 msec 2024-11-10T09:18:29,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T09:18:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:29,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/289de62e3ce34911aeaef37cc2f2f676 is 50, key is test_row_0/A:col10/1731230309602/Put/seqid=0 2024-11-10T09:18:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230369638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230369651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230369657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742379_1555 (size=14741) 2024-11-10T09:18:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230369768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230369777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230369777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:18:29,874 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:29,876 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-10T09:18:29,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:29,882 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:29,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:30,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230369997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:30,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230369998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:30,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230369998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:30,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:30,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:18:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=730 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/289de62e3ce34911aeaef37cc2f2f676 2024-11-10T09:18:30,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/2915f047af6b4a36b9d0ccc595e52f6a is 50, key is test_row_0/B:col10/1731230309602/Put/seqid=0 2024-11-10T09:18:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742380_1556 (size=12301) 2024-11-10T09:18:30,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=730 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/2915f047af6b4a36b9d0ccc595e52f6a 2024-11-10T09:18:30,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:18:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/50e47980c0dd42d39e51a54a12ce30bf is 50, key is test_row_0/C:col10/1731230309602/Put/seqid=0 2024-11-10T09:18:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:30,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742381_1557 (size=12301) 2024-11-10T09:18:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38412 deadline: 1731230370328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:30,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38454 deadline: 1731230370328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:30,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:18:30,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38470 deadline: 1731230370337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., hostname=802346a334f5,38513,1731230168538, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:18:30,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:18:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:30,514 INFO [Thread-2097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2097. 2024-11-10T09:18:30,514 DEBUG [Thread-2097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,514 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,514 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,522 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-11-10T09:18:30,522 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,522 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,522 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,524 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-11-10T09:18:30,524 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,524 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,524 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,525 INFO [Thread-2099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2099. 2024-11-10T09:18:30,525 DEBUG [Thread-2099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,525 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,525 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,533 INFO [Thread-2101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2101. 2024-11-10T09:18:30,533 DEBUG [Thread-2101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,533 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,533 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=730 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/50e47980c0dd42d39e51a54a12ce30bf 2024-11-10T09:18:30,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:18:30,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:30,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/289de62e3ce34911aeaef37cc2f2f676 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/289de62e3ce34911aeaef37cc2f2f676 2024-11-10T09:18:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:30,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/289de62e3ce34911aeaef37cc2f2f676, entries=200, sequenceid=730, filesize=14.4 K 2024-11-10T09:18:30,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/2915f047af6b4a36b9d0ccc595e52f6a as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/2915f047af6b4a36b9d0ccc595e52f6a 2024-11-10T09:18:30,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/2915f047af6b4a36b9d0ccc595e52f6a, entries=150, sequenceid=730, filesize=12.0 K 2024-11-10T09:18:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/50e47980c0dd42d39e51a54a12ce30bf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/50e47980c0dd42d39e51a54a12ce30bf 2024-11-10T09:18:30,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/50e47980c0dd42d39e51a54a12ce30bf, entries=150, sequenceid=730, filesize=12.0 K 2024-11-10T09:18:30,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5b0c14639876e70a158c7f1b205d25b6 in 1063ms, sequenceid=730, compaction requested=true 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:30,666 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b0c14639876e70a158c7f1b205d25b6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:30,666 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:30,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:30,667 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:30,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/A is initiating minor compaction (all files) 2024-11-10T09:18:30,667 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/B is initiating minor compaction (all files) 2024-11-10T09:18:30,667 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/A in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,667 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/B in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,667 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4b1e8fb4754e48b7a6525ac41363864f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b525c3f07e3b4581a6cfafd784f0a44c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/289de62e3ce34911aeaef37cc2f2f676] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=39.3 K 2024-11-10T09:18:30,667 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/78dca70a8d524edaaadbf39e263d873d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/475d5ec271334c7997d4c7c6399cd822, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/2915f047af6b4a36b9d0ccc595e52f6a] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.9 K 2024-11-10T09:18:30,667 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1e8fb4754e48b7a6525ac41363864f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731230307840 2024-11-10T09:18:30,667 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 78dca70a8d524edaaadbf39e263d873d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731230307840 2024-11-10T09:18:30,668 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 475d5ec271334c7997d4c7c6399cd822, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=707, earliestPutTs=1731230308524 2024-11-10T09:18:30,668 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting b525c3f07e3b4581a6cfafd784f0a44c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=707, earliestPutTs=1731230308524 2024-11-10T09:18:30,668 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 2915f047af6b4a36b9d0ccc595e52f6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=730, earliestPutTs=1731230309260 2024-11-10T09:18:30,668 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289de62e3ce34911aeaef37cc2f2f676, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=730, earliestPutTs=1731230309241 2024-11-10T09:18:30,674 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#A#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:30,674 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#B#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:30,674 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/35c047eb097c4848b72bb4eaee772325 is 50, key is test_row_0/B:col10/1731230309602/Put/seqid=0 2024-11-10T09:18:30,674 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/e4c8dbce6e654cb4b3c6ebf65b82eecf is 50, key is test_row_0/A:col10/1731230309602/Put/seqid=0 2024-11-10T09:18:30,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742383_1559 (size=13323) 2024-11-10T09:18:30,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742382_1558 (size=13323) 2024-11-10T09:18:30,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:18:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:30,808 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:18:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:30,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4cac131b06c54065aadd46f9a1c8d354 is 50, key is test_row_0/A:col10/1731230309636/Put/seqid=0 2024-11-10T09:18:30,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742384_1560 (size=12301) 2024-11-10T09:18:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. as already flushing 2024-11-10T09:18:30,848 INFO [Thread-2088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2088. 2024-11-10T09:18:30,848 INFO [Thread-2086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2086. 2024-11-10T09:18:30,848 DEBUG [Thread-2088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,848 DEBUG [Thread-2086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,848 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,848 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,848 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,848 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:30,868 INFO [Thread-2090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2090. 2024-11-10T09:18:30,868 DEBUG [Thread-2090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:30,869 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,869 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:30,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:31,081 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/35c047eb097c4848b72bb4eaee772325 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/35c047eb097c4848b72bb4eaee772325 2024-11-10T09:18:31,081 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/e4c8dbce6e654cb4b3c6ebf65b82eecf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e4c8dbce6e654cb4b3c6ebf65b82eecf 2024-11-10T09:18:31,084 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/A of 5b0c14639876e70a158c7f1b205d25b6 into e4c8dbce6e654cb4b3c6ebf65b82eecf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:31,084 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/B of 5b0c14639876e70a158c7f1b205d25b6 into 35c047eb097c4848b72bb4eaee772325(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:31,084 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/A, priority=13, startTime=1731230310666; duration=0sec 2024-11-10T09:18:31,084 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/B, priority=13, startTime=1731230310666; duration=0sec 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:A 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:B 2024-11-10T09:18:31,084 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:31,085 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:31,085 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 5b0c14639876e70a158c7f1b205d25b6/C is initiating minor compaction (all files) 2024-11-10T09:18:31,085 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b0c14639876e70a158c7f1b205d25b6/C in TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:31,085 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/929952d748f24e68b1e7533c2dd5e6a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/1fc5e2098795486da125c60edee6fda1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/50e47980c0dd42d39e51a54a12ce30bf] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp, totalSize=36.9 K 2024-11-10T09:18:31,085 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929952d748f24e68b1e7533c2dd5e6a7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731230307840 2024-11-10T09:18:31,085 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fc5e2098795486da125c60edee6fda1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=707, earliestPutTs=1731230308524 2024-11-10T09:18:31,086 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e47980c0dd42d39e51a54a12ce30bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=730, earliestPutTs=1731230309260 2024-11-10T09:18:31,097 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b0c14639876e70a158c7f1b205d25b6#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:31,097 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/72b61a672cfb49af8710266db989dd3d is 50, key is test_row_0/C:col10/1731230309602/Put/seqid=0 2024-11-10T09:18:31,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742385_1561 (size=13323) 2024-11-10T09:18:31,214 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4cac131b06c54065aadd46f9a1c8d354 2024-11-10T09:18:31,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7cd7e0b6a1fc420cbfe0f96569709844 is 50, key is test_row_0/B:col10/1731230309636/Put/seqid=0 2024-11-10T09:18:31,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742386_1562 (size=12301) 2024-11-10T09:18:31,504 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/72b61a672cfb49af8710266db989dd3d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/72b61a672cfb49af8710266db989dd3d 2024-11-10T09:18:31,507 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b0c14639876e70a158c7f1b205d25b6/C of 5b0c14639876e70a158c7f1b205d25b6 into 72b61a672cfb49af8710266db989dd3d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:31,507 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:31,507 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., storeName=5b0c14639876e70a158c7f1b205d25b6/C, priority=13, startTime=1731230310666; duration=0sec 2024-11-10T09:18:31,507 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:31,507 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b0c14639876e70a158c7f1b205d25b6:C 2024-11-10T09:18:31,624 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7cd7e0b6a1fc420cbfe0f96569709844 2024-11-10T09:18:31,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/40028fc5855e436e825300311fda6d48 is 50, key is test_row_0/C:col10/1731230309636/Put/seqid=0 2024-11-10T09:18:31,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742387_1563 (size=12301) 2024-11-10T09:18:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:32,033 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/40028fc5855e436e825300311fda6d48 2024-11-10T09:18:32,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/4cac131b06c54065aadd46f9a1c8d354 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4cac131b06c54065aadd46f9a1c8d354 2024-11-10T09:18:32,038 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4cac131b06c54065aadd46f9a1c8d354, entries=150, sequenceid=741, filesize=12.0 K 2024-11-10T09:18:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/7cd7e0b6a1fc420cbfe0f96569709844 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7cd7e0b6a1fc420cbfe0f96569709844 2024-11-10T09:18:32,041 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7cd7e0b6a1fc420cbfe0f96569709844, entries=150, sequenceid=741, filesize=12.0 K 2024-11-10T09:18:32,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/40028fc5855e436e825300311fda6d48 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/40028fc5855e436e825300311fda6d48 2024-11-10T09:18:32,044 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/40028fc5855e436e825300311fda6d48, entries=150, sequenceid=741, filesize=12.0 K 2024-11-10T09:18:32,045 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 5b0c14639876e70a158c7f1b205d25b6 in 1237ms, sequenceid=741, compaction requested=false 2024-11-10T09:18:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 5b0c14639876e70a158c7f1b205d25b6: 2024-11-10T09:18:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-10T09:18:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-10T09:18:32,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T09:18:32,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1640 sec 2024-11-10T09:18:32,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.1710 sec 2024-11-10T09:18:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:18:34,038 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:36,805 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:18:38,989 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-10T09:18:38,989 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:38,989 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:38,989 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:38,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:39,038 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-10T09:18:39,038 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:18:39,038 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:39,038 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:39,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:39,038 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 202 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 211 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 213 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 988 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2964 rows 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 991 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2970 rows 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 986 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2953 rows 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 992 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2973 rows 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 996 2024-11-10T09:18:39,039 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2988 rows 2024-11-10T09:18:39,039 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:18:39,039 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:18:39,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:39,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:39,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:18:39,040 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:18:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:18:39,042 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230319042"}]},"ts":"1731230319042"} 2024-11-10T09:18:39,043 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:18:39,043 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:18:39,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:18:39,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, UNASSIGN}] 2024-11-10T09:18:39,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, UNASSIGN 2024-11-10T09:18:39,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=5b0c14639876e70a158c7f1b205d25b6, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:39,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, UNASSIGN because future has completed 2024-11-10T09:18:39,047 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:18:39,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:18:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:18:39,199 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:39,199 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:18:39,199 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 5b0c14639876e70a158c7f1b205d25b6, disabling compactions & flushes 2024-11-10T09:18:39,199 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:39,199 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. after waiting 0 ms 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:39,200 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 5b0c14639876e70a158c7f1b205d25b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=A 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=B 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5b0c14639876e70a158c7f1b205d25b6, store=C 2024-11-10T09:18:39,200 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:39,203 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/af2d7c151a77469494402d879eb7d7a9 is 50, key is test_row_0/A:col10/1731230310847/Put/seqid=0 2024-11-10T09:18:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742388_1564 (size=12301) 2024-11-10T09:18:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:18:39,607 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=752 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/af2d7c151a77469494402d879eb7d7a9 2024-11-10T09:18:39,612 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/4d6955dfa63448cb8e95452fcbad6ed7 is 50, key is test_row_0/B:col10/1731230310847/Put/seqid=0 2024-11-10T09:18:39,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742389_1565 (size=12301) 2024-11-10T09:18:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:18:40,016 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=752 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/4d6955dfa63448cb8e95452fcbad6ed7 2024-11-10T09:18:40,020 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/41674ae9e4424cc88d93f6e8c9cf2a86 is 50, key is test_row_0/C:col10/1731230310847/Put/seqid=0 2024-11-10T09:18:40,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742390_1566 (size=12301) 2024-11-10T09:18:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:18:40,423 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=752 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/41674ae9e4424cc88d93f6e8c9cf2a86 2024-11-10T09:18:40,426 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/A/af2d7c151a77469494402d879eb7d7a9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/af2d7c151a77469494402d879eb7d7a9 2024-11-10T09:18:40,429 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/af2d7c151a77469494402d879eb7d7a9, entries=150, sequenceid=752, filesize=12.0 K 2024-11-10T09:18:40,429 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/B/4d6955dfa63448cb8e95452fcbad6ed7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/4d6955dfa63448cb8e95452fcbad6ed7 2024-11-10T09:18:40,432 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/4d6955dfa63448cb8e95452fcbad6ed7, entries=150, sequenceid=752, filesize=12.0 K 2024-11-10T09:18:40,432 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/.tmp/C/41674ae9e4424cc88d93f6e8c9cf2a86 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/41674ae9e4424cc88d93f6e8c9cf2a86 2024-11-10T09:18:40,435 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/41674ae9e4424cc88d93f6e8c9cf2a86, entries=150, sequenceid=752, filesize=12.0 K 2024-11-10T09:18:40,436 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5b0c14639876e70a158c7f1b205d25b6 in 1236ms, sequenceid=752, compaction requested=true 2024-11-10T09:18:40,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e338d06988064b49860390eba1f417da, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/5d3aeffc27b746d2a8c60264ee1f8d5f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9bffaffdbf3f415f943aec97e5ed76e5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9492117dd4c849a6b280213a37021168, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/741b49eaf54f402ea663bb19db86e3a1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/1459f49f9208434b83196e4efdd1a2ae, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4eab64a6c3d84de6a52a4e8479ecf312, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b0abdf2a87854979a485fda53dc5aea8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/44863c03fcf74350ba9b8bc1a4ab3d0b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/c397af195e7041a89f5d4c90b848e7cc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/3866056979dd4f8ca13428788d3d3a94, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b78fc1c742c5488abcaa39620dc0b7a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d10221d31360494aaa55016751e510ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/34ab4f385aa445e48a3c3b0dd1333010, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/92148bb348ee46178ca4be966aab2136, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ece408fd7fe746e7be94498242d42d9a, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ff46c5490f084283a40f56f795b4e34e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/910afaa75f02477194d5ac89bcc38d80, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/40a6561e87b5497f8156c73ec18326b4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/53dc12d29f084a1c9e79d015a20233f0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/98f61f2c2d06463c9c507a74c2bd0d30, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/852f858a09494f12b6b4963cd08c31bb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/51d14c8207a345ed8ac68e753e32af3f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ba35a2bf69364dd68d39c4b8b1f48a13, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/2da46de98b0e4bbbb54b745229c1f0b9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d7149b31e4604c7592971b848d2f6ae1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/6c47bd9a667846c0be762a33b784a780, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4b1e8fb4754e48b7a6525ac41363864f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b525c3f07e3b4581a6cfafd784f0a44c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/289de62e3ce34911aeaef37cc2f2f676] to archive 2024-11-10T09:18:40,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:18:40,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e338d06988064b49860390eba1f417da to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e338d06988064b49860390eba1f417da 2024-11-10T09:18:40,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/5d3aeffc27b746d2a8c60264ee1f8d5f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/5d3aeffc27b746d2a8c60264ee1f8d5f 2024-11-10T09:18:40,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9bffaffdbf3f415f943aec97e5ed76e5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9bffaffdbf3f415f943aec97e5ed76e5 2024-11-10T09:18:40,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9492117dd4c849a6b280213a37021168 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/9492117dd4c849a6b280213a37021168 2024-11-10T09:18:40,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/741b49eaf54f402ea663bb19db86e3a1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/741b49eaf54f402ea663bb19db86e3a1 2024-11-10T09:18:40,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/1459f49f9208434b83196e4efdd1a2ae to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/1459f49f9208434b83196e4efdd1a2ae 2024-11-10T09:18:40,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4eab64a6c3d84de6a52a4e8479ecf312 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4eab64a6c3d84de6a52a4e8479ecf312 2024-11-10T09:18:40,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b0abdf2a87854979a485fda53dc5aea8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b0abdf2a87854979a485fda53dc5aea8 2024-11-10T09:18:40,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/44863c03fcf74350ba9b8bc1a4ab3d0b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/44863c03fcf74350ba9b8bc1a4ab3d0b 2024-11-10T09:18:40,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/c397af195e7041a89f5d4c90b848e7cc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/c397af195e7041a89f5d4c90b848e7cc 2024-11-10T09:18:40,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/3866056979dd4f8ca13428788d3d3a94 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/3866056979dd4f8ca13428788d3d3a94 2024-11-10T09:18:40,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b78fc1c742c5488abcaa39620dc0b7a7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b78fc1c742c5488abcaa39620dc0b7a7 2024-11-10T09:18:40,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d10221d31360494aaa55016751e510ed to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d10221d31360494aaa55016751e510ed 2024-11-10T09:18:40,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/34ab4f385aa445e48a3c3b0dd1333010 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/34ab4f385aa445e48a3c3b0dd1333010 2024-11-10T09:18:40,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/92148bb348ee46178ca4be966aab2136 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/92148bb348ee46178ca4be966aab2136 2024-11-10T09:18:40,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ece408fd7fe746e7be94498242d42d9a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ece408fd7fe746e7be94498242d42d9a 2024-11-10T09:18:40,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ff46c5490f084283a40f56f795b4e34e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ff46c5490f084283a40f56f795b4e34e 2024-11-10T09:18:40,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/910afaa75f02477194d5ac89bcc38d80 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/910afaa75f02477194d5ac89bcc38d80 2024-11-10T09:18:40,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/40a6561e87b5497f8156c73ec18326b4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/40a6561e87b5497f8156c73ec18326b4 2024-11-10T09:18:40,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/53dc12d29f084a1c9e79d015a20233f0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/53dc12d29f084a1c9e79d015a20233f0 2024-11-10T09:18:40,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/98f61f2c2d06463c9c507a74c2bd0d30 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/98f61f2c2d06463c9c507a74c2bd0d30 2024-11-10T09:18:40,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/852f858a09494f12b6b4963cd08c31bb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/852f858a09494f12b6b4963cd08c31bb 2024-11-10T09:18:40,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/51d14c8207a345ed8ac68e753e32af3f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/51d14c8207a345ed8ac68e753e32af3f 2024-11-10T09:18:40,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ba35a2bf69364dd68d39c4b8b1f48a13 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/ba35a2bf69364dd68d39c4b8b1f48a13 2024-11-10T09:18:40,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/2da46de98b0e4bbbb54b745229c1f0b9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/2da46de98b0e4bbbb54b745229c1f0b9 2024-11-10T09:18:40,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d7149b31e4604c7592971b848d2f6ae1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/d7149b31e4604c7592971b848d2f6ae1 2024-11-10T09:18:40,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/6c47bd9a667846c0be762a33b784a780 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/6c47bd9a667846c0be762a33b784a780 2024-11-10T09:18:40,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4b1e8fb4754e48b7a6525ac41363864f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4b1e8fb4754e48b7a6525ac41363864f 2024-11-10T09:18:40,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b525c3f07e3b4581a6cfafd784f0a44c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/b525c3f07e3b4581a6cfafd784f0a44c 2024-11-10T09:18:40,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/289de62e3ce34911aeaef37cc2f2f676 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/289de62e3ce34911aeaef37cc2f2f676 2024-11-10T09:18:40,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50e95f60218d455fa84ebab90541b101, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ebf851be1536407a874f8ba161a1ef6f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50a45725a4374aad8d5c44ced379089b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5fdf7f9a59564b6ca093bf3a4a3c986c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/323f574a9d284f35bbf94c5620f7cd90, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/a5fef21ac72348ac9a52d7582f6f7638, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1111f6685dc34db399a9dea5efcbd66c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7be5a083191f4f20a0a31a97fe74f5ac, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/8e8064daed24455faadc88f7ba4fc551, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/823a61a7f43741bab298749ae7010e71, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/0cf17f442a80497791dda2e7c4a733bc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/10aed50b418d45deb5d1e8cf1b750a11, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/145379b8638d4097a8148f5ed015c352, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ece165a51d534f07b19fc0cb4ef30ab1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f04de1083dc143b5b693a1954ee75689, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1267d23a685949a7a49fe004f828087f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7ea2812a746547989334faebf86a3dbc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f7b5f865a5dd4670b811adc99e0dd2c1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/9b20064df8594738bbf8f9fbf6a43c50, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/382cdee5797a408d8c8bf7bf53cb4a1f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f11927768f3a4582b815c9eb1651e02e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5efedd7cf8944d0cbb49a315f883d988, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c65eb310bd824a059632c44e436f72ac, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c1d36a84ffbd4403b1881e87233887c4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/73ecd526600b4984a876ca4687455ffd, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/115aa146c05648c9a59920ace619a67b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/78dca70a8d524edaaadbf39e263d873d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/460ccc3273f14e8fa42420d87e447ecf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/475d5ec271334c7997d4c7c6399cd822, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/2915f047af6b4a36b9d0ccc595e52f6a] to archive 2024-11-10T09:18:40,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:18:40,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50e95f60218d455fa84ebab90541b101 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50e95f60218d455fa84ebab90541b101 2024-11-10T09:18:40,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ebf851be1536407a874f8ba161a1ef6f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ebf851be1536407a874f8ba161a1ef6f 2024-11-10T09:18:40,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50a45725a4374aad8d5c44ced379089b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/50a45725a4374aad8d5c44ced379089b 2024-11-10T09:18:40,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5fdf7f9a59564b6ca093bf3a4a3c986c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5fdf7f9a59564b6ca093bf3a4a3c986c 2024-11-10T09:18:40,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/323f574a9d284f35bbf94c5620f7cd90 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/323f574a9d284f35bbf94c5620f7cd90 2024-11-10T09:18:40,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/a5fef21ac72348ac9a52d7582f6f7638 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/a5fef21ac72348ac9a52d7582f6f7638 2024-11-10T09:18:40,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1111f6685dc34db399a9dea5efcbd66c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1111f6685dc34db399a9dea5efcbd66c 2024-11-10T09:18:40,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7be5a083191f4f20a0a31a97fe74f5ac to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7be5a083191f4f20a0a31a97fe74f5ac 2024-11-10T09:18:40,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/8e8064daed24455faadc88f7ba4fc551 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/8e8064daed24455faadc88f7ba4fc551 2024-11-10T09:18:40,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/823a61a7f43741bab298749ae7010e71 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/823a61a7f43741bab298749ae7010e71 2024-11-10T09:18:40,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/0cf17f442a80497791dda2e7c4a733bc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/0cf17f442a80497791dda2e7c4a733bc 2024-11-10T09:18:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/10aed50b418d45deb5d1e8cf1b750a11 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/10aed50b418d45deb5d1e8cf1b750a11 2024-11-10T09:18:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/145379b8638d4097a8148f5ed015c352 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/145379b8638d4097a8148f5ed015c352 2024-11-10T09:18:40,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ece165a51d534f07b19fc0cb4ef30ab1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/ece165a51d534f07b19fc0cb4ef30ab1 2024-11-10T09:18:40,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f04de1083dc143b5b693a1954ee75689 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f04de1083dc143b5b693a1954ee75689 2024-11-10T09:18:40,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1267d23a685949a7a49fe004f828087f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/1267d23a685949a7a49fe004f828087f 2024-11-10T09:18:40,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7ea2812a746547989334faebf86a3dbc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7ea2812a746547989334faebf86a3dbc 2024-11-10T09:18:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f7b5f865a5dd4670b811adc99e0dd2c1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f7b5f865a5dd4670b811adc99e0dd2c1 2024-11-10T09:18:40,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/9b20064df8594738bbf8f9fbf6a43c50 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/9b20064df8594738bbf8f9fbf6a43c50 2024-11-10T09:18:40,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/382cdee5797a408d8c8bf7bf53cb4a1f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/382cdee5797a408d8c8bf7bf53cb4a1f 2024-11-10T09:18:40,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f11927768f3a4582b815c9eb1651e02e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/f11927768f3a4582b815c9eb1651e02e 2024-11-10T09:18:40,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5efedd7cf8944d0cbb49a315f883d988 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/5efedd7cf8944d0cbb49a315f883d988 2024-11-10T09:18:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c65eb310bd824a059632c44e436f72ac to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c65eb310bd824a059632c44e436f72ac 2024-11-10T09:18:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c1d36a84ffbd4403b1881e87233887c4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/c1d36a84ffbd4403b1881e87233887c4 2024-11-10T09:18:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/73ecd526600b4984a876ca4687455ffd to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/73ecd526600b4984a876ca4687455ffd 2024-11-10T09:18:40,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/115aa146c05648c9a59920ace619a67b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/115aa146c05648c9a59920ace619a67b 2024-11-10T09:18:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/78dca70a8d524edaaadbf39e263d873d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/78dca70a8d524edaaadbf39e263d873d 2024-11-10T09:18:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/460ccc3273f14e8fa42420d87e447ecf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/460ccc3273f14e8fa42420d87e447ecf 2024-11-10T09:18:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/475d5ec271334c7997d4c7c6399cd822 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/475d5ec271334c7997d4c7c6399cd822 2024-11-10T09:18:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/2915f047af6b4a36b9d0ccc595e52f6a to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/2915f047af6b4a36b9d0ccc595e52f6a 2024-11-10T09:18:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/37a7373395144e63a4b15505926879aa, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/213849e1625d416691bc2fc9757f550d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/e14cb508336943d4a18b9106bc105768, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/cd1141c4e3494df28562f3b340276146, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f8f76a4116a74831b1ee7f8844777a5f, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/55cc37953757435e9ddd9cfe2ad930f1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/32f17da07c7b4e52abe290db0ba85b42, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4d7abde2d72d427abfffa766e36d9c29, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5bbdac4f9fa242b2b97ae94ba8996c5d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/a1be407704484d44964143cfa959bd6c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2454c8be228e4c9db63eb18881a76320, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/546fec6311474634bab3c0b3e889b6dc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/78bf253810e34bc2aae33e7bc5a8a631, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f434bfeb9488417882c43a552d7ab4a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/d67037481609401890f1041c8a741668, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/7ae3665d93974a4e954b089aba9c560d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4a074b41d2ff4d4b8ce171946e392c52, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5eff7e628de44c35abdc2e1892fa1f55, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/96dea96b8648414da75dcbe6cc29d2c9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c73b1cd5f61c4111a5bd3b190340b5f1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c8a0b1fb61fb4163bce9f6750655a126, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5c0f3740c0984100a2d641b365998eb3, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5215de822d38475db7fc9c2609214c45, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/35e6ef5584204edd986beceb9c33feea, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/758b0451fa2a478ab951d299e3f33b51, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2fce20df17274527b65d6d43d84c57e5, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/929952d748f24e68b1e7533c2dd5e6a7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/73f4f042c1544054b1bf1d739f0481a9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/1fc5e2098795486da125c60edee6fda1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/50e47980c0dd42d39e51a54a12ce30bf] to archive 2024-11-10T09:18:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:18:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/37a7373395144e63a4b15505926879aa to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/37a7373395144e63a4b15505926879aa 2024-11-10T09:18:40,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/213849e1625d416691bc2fc9757f550d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/213849e1625d416691bc2fc9757f550d 2024-11-10T09:18:40,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/e14cb508336943d4a18b9106bc105768 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/e14cb508336943d4a18b9106bc105768 2024-11-10T09:18:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/cd1141c4e3494df28562f3b340276146 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/cd1141c4e3494df28562f3b340276146 2024-11-10T09:18:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f8f76a4116a74831b1ee7f8844777a5f to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f8f76a4116a74831b1ee7f8844777a5f 2024-11-10T09:18:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/55cc37953757435e9ddd9cfe2ad930f1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/55cc37953757435e9ddd9cfe2ad930f1 2024-11-10T09:18:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/32f17da07c7b4e52abe290db0ba85b42 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/32f17da07c7b4e52abe290db0ba85b42 2024-11-10T09:18:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4d7abde2d72d427abfffa766e36d9c29 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4d7abde2d72d427abfffa766e36d9c29 2024-11-10T09:18:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5bbdac4f9fa242b2b97ae94ba8996c5d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5bbdac4f9fa242b2b97ae94ba8996c5d 2024-11-10T09:18:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/a1be407704484d44964143cfa959bd6c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/a1be407704484d44964143cfa959bd6c 2024-11-10T09:18:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2454c8be228e4c9db63eb18881a76320 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2454c8be228e4c9db63eb18881a76320 2024-11-10T09:18:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/546fec6311474634bab3c0b3e889b6dc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/546fec6311474634bab3c0b3e889b6dc 2024-11-10T09:18:40,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/78bf253810e34bc2aae33e7bc5a8a631 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/78bf253810e34bc2aae33e7bc5a8a631 2024-11-10T09:18:40,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f434bfeb9488417882c43a552d7ab4a7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/f434bfeb9488417882c43a552d7ab4a7 2024-11-10T09:18:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/d67037481609401890f1041c8a741668 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/d67037481609401890f1041c8a741668 2024-11-10T09:18:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/7ae3665d93974a4e954b089aba9c560d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/7ae3665d93974a4e954b089aba9c560d 2024-11-10T09:18:40,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4a074b41d2ff4d4b8ce171946e392c52 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/4a074b41d2ff4d4b8ce171946e392c52 2024-11-10T09:18:40,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5eff7e628de44c35abdc2e1892fa1f55 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5eff7e628de44c35abdc2e1892fa1f55 2024-11-10T09:18:40,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/96dea96b8648414da75dcbe6cc29d2c9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/96dea96b8648414da75dcbe6cc29d2c9 2024-11-10T09:18:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c73b1cd5f61c4111a5bd3b190340b5f1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c73b1cd5f61c4111a5bd3b190340b5f1 2024-11-10T09:18:40,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c8a0b1fb61fb4163bce9f6750655a126 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/c8a0b1fb61fb4163bce9f6750655a126 2024-11-10T09:18:40,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5c0f3740c0984100a2d641b365998eb3 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5c0f3740c0984100a2d641b365998eb3 2024-11-10T09:18:40,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5215de822d38475db7fc9c2609214c45 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/5215de822d38475db7fc9c2609214c45 2024-11-10T09:18:40,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/35e6ef5584204edd986beceb9c33feea to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/35e6ef5584204edd986beceb9c33feea 2024-11-10T09:18:40,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/758b0451fa2a478ab951d299e3f33b51 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/758b0451fa2a478ab951d299e3f33b51 2024-11-10T09:18:40,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2fce20df17274527b65d6d43d84c57e5 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/2fce20df17274527b65d6d43d84c57e5 2024-11-10T09:18:40,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/929952d748f24e68b1e7533c2dd5e6a7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/929952d748f24e68b1e7533c2dd5e6a7 2024-11-10T09:18:40,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/73f4f042c1544054b1bf1d739f0481a9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/73f4f042c1544054b1bf1d739f0481a9 2024-11-10T09:18:40,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/1fc5e2098795486da125c60edee6fda1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/1fc5e2098795486da125c60edee6fda1 2024-11-10T09:18:40,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/50e47980c0dd42d39e51a54a12ce30bf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/50e47980c0dd42d39e51a54a12ce30bf 2024-11-10T09:18:40,509 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/recovered.edits/755.seqid, newMaxSeqId=755, maxSeqId=1 2024-11-10T09:18:40,510 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. 2024-11-10T09:18:40,510 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 5b0c14639876e70a158c7f1b205d25b6: Waiting for close lock at 1731230319199Running coprocessor pre-close hooks at 1731230319199Disabling compacts and flushes for region at 1731230319199Disabling writes for close at 1731230319200 (+1 ms)Obtaining lock to block concurrent updates at 1731230319200Preparing flush snapshotting stores in 5b0c14639876e70a158c7f1b205d25b6 at 1731230319200Finished memstore snapshotting TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731230319200Flushing stores of TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6. at 1731230319200Flushing 5b0c14639876e70a158c7f1b205d25b6/A: creating writer at 1731230319200Flushing 5b0c14639876e70a158c7f1b205d25b6/A: appending metadata at 1731230319203 (+3 ms)Flushing 5b0c14639876e70a158c7f1b205d25b6/A: closing flushed file at 1731230319203Flushing 5b0c14639876e70a158c7f1b205d25b6/B: creating writer at 1731230319610 (+407 ms)Flushing 5b0c14639876e70a158c7f1b205d25b6/B: appending metadata at 1731230319612 (+2 ms)Flushing 5b0c14639876e70a158c7f1b205d25b6/B: closing flushed file at 1731230319612Flushing 5b0c14639876e70a158c7f1b205d25b6/C: creating writer at 1731230320018 (+406 ms)Flushing 5b0c14639876e70a158c7f1b205d25b6/C: appending metadata at 1731230320020 (+2 ms)Flushing 5b0c14639876e70a158c7f1b205d25b6/C: closing flushed file at 1731230320020Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36699fe9: reopening flushed file at 1731230320426 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27c93813: reopening flushed file at 1731230320429 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66b39385: reopening flushed file at 1731230320432 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5b0c14639876e70a158c7f1b205d25b6 in 1236ms, sequenceid=752, compaction requested=true at 1731230320436 (+4 ms)Writing region close event to WAL at 1731230320507 (+71 ms)Running coprocessor post-close hooks at 1731230320510 (+3 ms)Closed at 1731230320510 2024-11-10T09:18:40,511 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:40,511 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=5b0c14639876e70a158c7f1b205d25b6, regionState=CLOSED 2024-11-10T09:18:40,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:18:40,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T09:18:40,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 5b0c14639876e70a158c7f1b205d25b6, server=802346a334f5,38513,1731230168538 in 1.4660 sec 2024-11-10T09:18:40,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-10T09:18:40,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b0c14639876e70a158c7f1b205d25b6, UNASSIGN in 1.4690 sec 2024-11-10T09:18:40,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T09:18:40,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-11-10T09:18:40,518 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230320518"}]},"ts":"1731230320518"} 2024-11-10T09:18:40,519 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:18:40,519 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:18:40,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-10T09:18:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:18:41,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:41,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:18:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,189 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T09:18:41,190 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,191 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:41,193 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/recovered.edits] 2024-11-10T09:18:41,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4cac131b06c54065aadd46f9a1c8d354 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/4cac131b06c54065aadd46f9a1c8d354 2024-11-10T09:18:41,195 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/af2d7c151a77469494402d879eb7d7a9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/af2d7c151a77469494402d879eb7d7a9 2024-11-10T09:18:41,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e4c8dbce6e654cb4b3c6ebf65b82eecf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/A/e4c8dbce6e654cb4b3c6ebf65b82eecf 2024-11-10T09:18:41,197 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/35c047eb097c4848b72bb4eaee772325 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/35c047eb097c4848b72bb4eaee772325 2024-11-10T09:18:41,198 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/4d6955dfa63448cb8e95452fcbad6ed7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/4d6955dfa63448cb8e95452fcbad6ed7 2024-11-10T09:18:41,199 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7cd7e0b6a1fc420cbfe0f96569709844 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/B/7cd7e0b6a1fc420cbfe0f96569709844 2024-11-10T09:18:41,201 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/40028fc5855e436e825300311fda6d48 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/40028fc5855e436e825300311fda6d48 2024-11-10T09:18:41,201 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/41674ae9e4424cc88d93f6e8c9cf2a86 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/41674ae9e4424cc88d93f6e8c9cf2a86 2024-11-10T09:18:41,202 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/72b61a672cfb49af8710266db989dd3d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/C/72b61a672cfb49af8710266db989dd3d 2024-11-10T09:18:41,204 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/recovered.edits/755.seqid to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6/recovered.edits/755.seqid 2024-11-10T09:18:41,204 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/5b0c14639876e70a158c7f1b205d25b6 2024-11-10T09:18:41,204 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:18:41,206 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,208 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:18:41,209 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:18:41,210 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,210 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:18:41,210 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230321210"}]},"ts":"9223372036854775807"} 2024-11-10T09:18:41,212 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:18:41,212 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5b0c14639876e70a158c7f1b205d25b6, NAME => 'TestAcidGuarantees,,1731230288295.5b0c14639876e70a158c7f1b205d25b6.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:18:41,212 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:18:41,212 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230321212"}]},"ts":"9223372036854775807"} 2024-11-10T09:18:41,213 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:18:41,214 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-10T09:18:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T09:18:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:18:41,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:41,307 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 251), OpenFileDescriptor=443 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=747 (was 796), ProcessCount=11 (was 11), AvailableMemoryMB=1515 (was 2387) 2024-11-10T09:18:41,315 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=747, ProcessCount=11, AvailableMemoryMB=1514 2024-11-10T09:18:41,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:18:41,316 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:18:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:41,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:18:41,317 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:41,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-10T09:18:41,318 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:18:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:18:41,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742391_1567 (size=1061) 2024-11-10T09:18:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:18:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:18:41,724 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11 2024-11-10T09:18:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742392_1568 (size=53) 2024-11-10T09:18:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:18:42,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:18:42,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 38741dfb7340c00e6e9f2028384a14b8, disabling compactions & flushes 2024-11-10T09:18:42,128 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. after waiting 0 ms 2024-11-10T09:18:42,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,128 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38741dfb7340c00e6e9f2028384a14b8: Waiting for close lock at 1731230322128Disabling compacts and flushes for region at 1731230322128Disabling writes for close at 1731230322128Writing region close event to WAL at 1731230322128Closed at 1731230322128 2024-11-10T09:18:42,129 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:18:42,130 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230322129"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230322129"}]},"ts":"1731230322129"} 2024-11-10T09:18:42,131 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:18:42,131 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:18:42,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230322131"}]},"ts":"1731230322131"} 2024-11-10T09:18:42,132 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:18:42,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, ASSIGN}] 2024-11-10T09:18:42,133 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, ASSIGN 2024-11-10T09:18:42,133 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, ASSIGN; state=OFFLINE, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=false 2024-11-10T09:18:42,284 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:42,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, ASSIGN because future has completed 2024-11-10T09:18:42,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:18:42,439 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,439 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:18:42,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:18:42,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,440 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,441 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,442 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:42,442 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38741dfb7340c00e6e9f2028384a14b8 columnFamilyName A 2024-11-10T09:18:42,442 DEBUG [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:42,442 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(327): Store=38741dfb7340c00e6e9f2028384a14b8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:42,442 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,443 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:42,443 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38741dfb7340c00e6e9f2028384a14b8 columnFamilyName B 2024-11-10T09:18:42,443 DEBUG [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:42,444 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(327): Store=38741dfb7340c00e6e9f2028384a14b8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:42,444 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,444 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:42,444 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38741dfb7340c00e6e9f2028384a14b8 columnFamilyName C 2024-11-10T09:18:42,444 DEBUG [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:42,445 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(327): Store=38741dfb7340c00e6e9f2028384a14b8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:42,445 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,445 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,445 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,445 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,446 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,446 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,447 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:18:42,447 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,449 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:18:42,449 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 38741dfb7340c00e6e9f2028384a14b8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63589426, jitterRate=-0.05244371294975281}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:18:42,449 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:42,450 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 38741dfb7340c00e6e9f2028384a14b8: Running coprocessor pre-open hook at 1731230322440Writing region info on filesystem at 1731230322440Initializing all the Stores at 1731230322440Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230322441 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230322441Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230322441Cleaning up temporary data from old regions at 1731230322446 (+5 ms)Running coprocessor post-open hooks at 1731230322449 (+3 ms)Region opened successfully at 1731230322450 (+1 ms) 2024-11-10T09:18:42,451 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., pid=146, masterSystemTime=1731230322437 2024-11-10T09:18:42,452 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,452 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:42,452 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:42,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:18:42,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-10T09:18:42,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 in 168 msec 2024-11-10T09:18:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:18:42,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-10T09:18:42,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, ASSIGN in 324 msec 2024-11-10T09:18:42,457 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:18:42,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230322457"}]},"ts":"1731230322457"} 2024-11-10T09:18:42,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:18:42,459 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:18:42,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1430 sec 2024-11-10T09:18:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:18:43,468 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:43,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6e5d97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:43,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:43,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:43,470 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7ef3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:43,471 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5256d5af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:43,472 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:43,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:43,474 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:18:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,43719,1731230167759 2024-11-10T09:18:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22efbeea 2024-11-10T09:18:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:18:43,476 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:18:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:18:43,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:18:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T09:18:43,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742393_1569 (size=1097) 2024-11-10T09:18:43,887 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T09:18:43,887 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T09:18:43,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:18:43,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, REOPEN/MOVE}] 2024-11-10T09:18:43,892 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, REOPEN/MOVE 2024-11-10T09:18:43,892 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:43,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, REOPEN/MOVE because future has completed 2024-11-10T09:18:43,893 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:18:43,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:18:44,045 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,045 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:18:44,045 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 38741dfb7340c00e6e9f2028384a14b8, disabling compactions & flushes 2024-11-10T09:18:44,045 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,045 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,045 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. after waiting 0 ms 2024-11-10T09:18:44,045 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,048 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T09:18:44,049 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,049 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 38741dfb7340c00e6e9f2028384a14b8: Waiting for close lock at 1731230324045Running coprocessor pre-close hooks at 1731230324045Disabling compacts and flushes for region at 1731230324045Disabling writes for close at 1731230324045Writing region close event to WAL at 1731230324046 (+1 ms)Running coprocessor post-close hooks at 1731230324049 (+3 ms)Closed at 1731230324049 2024-11-10T09:18:44,049 WARN [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 38741dfb7340c00e6e9f2028384a14b8 to self. 2024-11-10T09:18:44,051 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,051 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=CLOSED 2024-11-10T09:18:44,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:18:44,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-10T09:18:44,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 in 160 msec 2024-11-10T09:18:44,055 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, REOPEN/MOVE; state=CLOSED, location=802346a334f5,38513,1731230168538; forceNewPlan=false, retain=true 2024-11-10T09:18:44,205 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=OPENING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, REOPEN/MOVE because future has completed 2024-11-10T09:18:44,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:18:44,360 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,361 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:18:44,361 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,361 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:18:44,361 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,361 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,362 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,362 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:44,363 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38741dfb7340c00e6e9f2028384a14b8 columnFamilyName A 2024-11-10T09:18:44,364 DEBUG [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:44,364 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(327): Store=38741dfb7340c00e6e9f2028384a14b8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:44,364 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,364 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:44,365 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38741dfb7340c00e6e9f2028384a14b8 columnFamilyName B 2024-11-10T09:18:44,365 DEBUG [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:44,365 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(327): Store=38741dfb7340c00e6e9f2028384a14b8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:44,365 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,365 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T09:18:44,366 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38741dfb7340c00e6e9f2028384a14b8 columnFamilyName C 2024-11-10T09:18:44,366 DEBUG [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:44,366 INFO [StoreOpener-38741dfb7340c00e6e9f2028384a14b8-1 {}] regionserver.HStore(327): Store=38741dfb7340c00e6e9f2028384a14b8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:18:44,366 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,366 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,367 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,367 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,368 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,368 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,368 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:18:44,369 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,370 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 38741dfb7340c00e6e9f2028384a14b8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60898019, jitterRate=-0.09254880249500275}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:18:44,370 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,370 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 38741dfb7340c00e6e9f2028384a14b8: Running coprocessor pre-open hook at 1731230324361Writing region info on filesystem at 1731230324361Initializing all the Stores at 1731230324362 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230324362Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230324362Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230324362Cleaning up temporary data from old regions at 1731230324368 (+6 ms)Running coprocessor post-open hooks at 1731230324370 (+2 ms)Region opened successfully at 1731230324370 2024-11-10T09:18:44,371 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., pid=151, masterSystemTime=1731230324358 2024-11-10T09:18:44,372 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,372 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,373 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=OPEN, openSeqNum=5, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:18:44,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-10T09:18:44,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 in 167 msec 2024-11-10T09:18:44,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T09:18:44,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, REOPEN/MOVE in 484 msec 2024-11-10T09:18:44,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-10T09:18:44,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-10T09:18:44,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-10T09:18:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T09:18:44,380 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:44,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfb61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2975a4b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65efef1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310d68c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22da7851, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df5819c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3341a350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64655441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,388 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d9c678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acdd241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,389 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564a7df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f85f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f076f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fac8f9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3721c259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22935e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,395 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eacc26f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a6df2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc7876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e6f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,398 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed6a607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e72bfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,400 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318c6db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48e5e4c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5a171f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c29837e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404fc931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7278e824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,43719,-1 for getting cluster id 2024-11-10T09:18:44,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:18:44,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a1f5e9f-ae4c-4e58-873b-13922a881c13' 2024-11-10T09:18:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:18:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a1f5e9f-ae4c-4e58-873b-13922a881c13" 2024-11-10T09:18:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52dbf437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,43719,-1] 2024-11-10T09:18:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:18:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:18:44,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:18:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38033399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:18:44,408 DEBUG [Thread-2534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,408 DEBUG [Thread-2536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,408 DEBUG [Thread-2542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,416 DEBUG [Thread-2538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,418 DEBUG [Thread-2523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,418 DEBUG [Thread-2525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:18:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:44,444 DEBUG [Thread-2527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,449 DEBUG [Thread-2529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230324454, max timestamp=1731230324463Num uniques 859; 2024-11-10T09:18:44,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:44,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230324454, max timestamp=1731230324462Num uniques 859; 2024-11-10T09:18:44,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:44,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230324454, max timestamp=1731230324463Num uniques 859; 2024-11-10T09:18:44,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:44,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-10T09:18:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:44,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:44,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:44,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230384483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230384485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230384486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230384487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,492 DEBUG [Thread-2531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230384496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a85ab10828ba41d18ac51b4b4dfd4354_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230324424/Put/seqid=0 2024-11-10T09:18:44,520 DEBUG [Thread-2540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:18:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,38513,1731230168538, seqNum=-1] 2024-11-10T09:18:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:18:44,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:18:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5] 2024-11-10T09:18:44,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742394_1570 (size=14594) 2024-11-10T09:18:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230384608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:44,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:44,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230384808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230384817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230384817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230384818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230384818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:44,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:44,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:44,950 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:44,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a85ab10828ba41d18ac51b4b4dfd4354_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a85ab10828ba41d18ac51b4b4dfd4354_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:44,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/adaa3d95c6d7498a974f964b4de21230, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:44,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/adaa3d95c6d7498a974f964b4de21230 is 175, key is test_row_0/A:col10/1731230324424/Put/seqid=0 2024-11-10T09:18:44,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742395_1571 (size=39549) 2024-11-10T09:18:44,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/adaa3d95c6d7498a974f964b4de21230 2024-11-10T09:18:44,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/c72849c73bed472da2158f5d7244d24c is 50, key is test_row_0/B:col10/1731230324424/Put/seqid=0 2024-11-10T09:18:44,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742396_1572 (size=12001) 2024-11-10T09:18:45,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230385117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230385127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230385127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230385127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230385127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/c72849c73bed472da2158f5d7244d24c 2024-11-10T09:18:45,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/1923b46a014f4db79cdefcd92a8b110e is 50, key is test_row_0/C:col10/1731230324424/Put/seqid=0 2024-11-10T09:18:45,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742397_1573 (size=12001) 2024-11-10T09:18:45,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230385627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230385637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230385637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230385637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230385637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:45,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:45,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/1923b46a014f4db79cdefcd92a8b110e 2024-11-10T09:18:45,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/adaa3d95c6d7498a974f964b4de21230 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230 2024-11-10T09:18:45,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230, entries=200, sequenceid=15, filesize=38.6 K 2024-11-10T09:18:45,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/c72849c73bed472da2158f5d7244d24c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/c72849c73bed472da2158f5d7244d24c 2024-11-10T09:18:45,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/c72849c73bed472da2158f5d7244d24c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:18:45,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/1923b46a014f4db79cdefcd92a8b110e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/1923b46a014f4db79cdefcd92a8b110e 2024-11-10T09:18:45,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/1923b46a014f4db79cdefcd92a8b110e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:18:45,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 38741dfb7340c00e6e9f2028384a14b8 in 1409ms, sequenceid=15, compaction requested=false 2024-11-10T09:18:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T09:18:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:45,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:45,855 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:45,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b66e4c3d65604127a59a848706459c31_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230324486/Put/seqid=0 2024-11-10T09:18:45,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742398_1574 (size=12154) 2024-11-10T09:18:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:45,915 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b66e4c3d65604127a59a848706459c31_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b66e4c3d65604127a59a848706459c31_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1f6e94249ba84d46a0dfe041b691f71d, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1f6e94249ba84d46a0dfe041b691f71d is 175, key is test_row_0/A:col10/1731230324486/Put/seqid=0 2024-11-10T09:18:45,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742399_1575 (size=30955) 2024-11-10T09:18:45,960 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1f6e94249ba84d46a0dfe041b691f71d 2024-11-10T09:18:45,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/7b4da0e73c9a4f15b19d45e4500ed2ba is 50, key is test_row_0/B:col10/1731230324486/Put/seqid=0 2024-11-10T09:18:45,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742400_1576 (size=12001) 2024-11-10T09:18:46,030 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:18:46,383 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/7b4da0e73c9a4f15b19d45e4500ed2ba 2024-11-10T09:18:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/7534425065a848f3a43a61f201b9d940 is 50, key is test_row_0/C:col10/1731230324486/Put/seqid=0 2024-11-10T09:18:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742401_1577 (size=12001) 2024-11-10T09:18:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:46,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:46,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230386656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230386656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230386656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230386657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230386657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,661 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230386767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230386767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230386767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230386767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:46,795 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/7534425065a848f3a43a61f201b9d940 2024-11-10T09:18:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1f6e94249ba84d46a0dfe041b691f71d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d 2024-11-10T09:18:46,802 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d, entries=150, sequenceid=50, filesize=30.2 K 2024-11-10T09:18:46,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/7b4da0e73c9a4f15b19d45e4500ed2ba as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/7b4da0e73c9a4f15b19d45e4500ed2ba 2024-11-10T09:18:46,806 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/7b4da0e73c9a4f15b19d45e4500ed2ba, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T09:18:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/7534425065a848f3a43a61f201b9d940 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7534425065a848f3a43a61f201b9d940 2024-11-10T09:18:46,810 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7534425065a848f3a43a61f201b9d940, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T09:18:46,811 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 38741dfb7340c00e6e9f2028384a14b8 in 955ms, sequenceid=50, compaction requested=false 2024-11-10T09:18:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-10T09:18:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-10T09:18:46,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T09:18:46,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-10T09:18:46,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.3370 sec 2024-11-10T09:18:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:46,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:18:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:46,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ea9e50e2da7e4286a0174ccd44d7ae35_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230326978/Put/seqid=0 2024-11-10T09:18:46,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742402_1578 (size=12154) 2024-11-10T09:18:47,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230387011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230387012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230387013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230387014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230387128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230387128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230387128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230387128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230387337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230387337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230387337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230387338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:47,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ea9e50e2da7e4286a0174ccd44d7ae35_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ea9e50e2da7e4286a0174ccd44d7ae35_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:47,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2cc7a84ebbd043b898ec2f3513981d45, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:47,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2cc7a84ebbd043b898ec2f3513981d45 is 175, key is test_row_0/A:col10/1731230326978/Put/seqid=0 2024-11-10T09:18:47,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742403_1579 (size=30955) 2024-11-10T09:18:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230387647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230387647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230387647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230387648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:47,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2cc7a84ebbd043b898ec2f3513981d45 2024-11-10T09:18:47,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/83e6df9961464553b0c6bd8d4d5c2fb0 is 50, key is test_row_0/B:col10/1731230326978/Put/seqid=0 2024-11-10T09:18:47,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742404_1580 (size=12001) 2024-11-10T09:18:47,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/83e6df9961464553b0c6bd8d4d5c2fb0 2024-11-10T09:18:47,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/0fe8e1830999474b902dc6c77fa89956 is 50, key is test_row_0/C:col10/1731230326978/Put/seqid=0 2024-11-10T09:18:47,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742405_1581 (size=12001) 2024-11-10T09:18:47,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/0fe8e1830999474b902dc6c77fa89956 2024-11-10T09:18:47,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2cc7a84ebbd043b898ec2f3513981d45 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45 2024-11-10T09:18:47,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45, entries=150, sequenceid=63, filesize=30.2 K 2024-11-10T09:18:47,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/83e6df9961464553b0c6bd8d4d5c2fb0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/83e6df9961464553b0c6bd8d4d5c2fb0 2024-11-10T09:18:47,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/83e6df9961464553b0c6bd8d4d5c2fb0, entries=150, sequenceid=63, filesize=11.7 K 2024-11-10T09:18:47,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/0fe8e1830999474b902dc6c77fa89956 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/0fe8e1830999474b902dc6c77fa89956 2024-11-10T09:18:47,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/0fe8e1830999474b902dc6c77fa89956, entries=150, sequenceid=63, filesize=11.7 K 2024-11-10T09:18:47,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 38741dfb7340c00e6e9f2028384a14b8 in 908ms, sequenceid=63, compaction requested=true 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:47,887 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:18:47,887 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:47,888 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:47,889 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:47,889 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:18:47,889 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:47,889 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/c72849c73bed472da2158f5d7244d24c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/7b4da0e73c9a4f15b19d45e4500ed2ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/83e6df9961464553b0c6bd8d4d5c2fb0] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=35.2 K 2024-11-10T09:18:47,889 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:18:47,889 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:47,889 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=99.1 K 2024-11-10T09:18:47,889 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:47,889 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45] 2024-11-10T09:18:47,890 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting adaa3d95c6d7498a974f964b4de21230, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230324424 2024-11-10T09:18:47,890 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting c72849c73bed472da2158f5d7244d24c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230324424 2024-11-10T09:18:47,890 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f6e94249ba84d46a0dfe041b691f71d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731230324480 2024-11-10T09:18:47,890 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4da0e73c9a4f15b19d45e4500ed2ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731230324480 2024-11-10T09:18:47,890 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cc7a84ebbd043b898ec2f3513981d45, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731230326655 2024-11-10T09:18:47,890 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e6df9961464553b0c6bd8d4d5c2fb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731230326655 2024-11-10T09:18:47,902 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:47,902 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d27876400f0d4a43add757778d6fa197 is 50, key is test_row_0/B:col10/1731230326978/Put/seqid=0 2024-11-10T09:18:47,913 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:47,933 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111020375c183a4441738f18c4caa6596750_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:47,934 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111020375c183a4441738f18c4caa6596750_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:47,934 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111020375c183a4441738f18c4caa6596750_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:47,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742406_1582 (size=12104) 2024-11-10T09:18:47,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742407_1583 (size=4469) 2024-11-10T09:18:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:48,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T09:18:48,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230327004, max timestamp=1731230328162Num uniques 859; 2024-11-10T09:18:48,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:48,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230327004, max timestamp=1731230328162Num uniques 859; 2024-11-10T09:18:48,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:18:48,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:48,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101da8c8e1a74e4f219b3f35f3a539631f_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230328170/Put/seqid=0 2024-11-10T09:18:48,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742408_1584 (size=14594) 2024-11-10T09:18:48,209 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:48,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101da8c8e1a74e4f219b3f35f3a539631f_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101da8c8e1a74e4f219b3f35f3a539631f_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:48,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0a1f5f9f73d745148e09a1a5e18f656d, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:48,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0a1f5f9f73d745148e09a1a5e18f656d is 175, key is test_row_0/A:col10/1731230328170/Put/seqid=0 2024-11-10T09:18:48,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230388208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742409_1585 (size=39549) 2024-11-10T09:18:48,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230388212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230388215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230388216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0a1f5f9f73d745148e09a1a5e18f656d 2024-11-10T09:18:48,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/085bc5ca9a084498aee8211074d9f8b8 is 50, key is test_row_0/B:col10/1731230328170/Put/seqid=0 2024-11-10T09:18:48,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742410_1586 (size=12001) 2024-11-10T09:18:48,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/085bc5ca9a084498aee8211074d9f8b8 2024-11-10T09:18:48,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/16b73c10ec7b4b6491308d03677fa0f6 is 50, key is test_row_0/C:col10/1731230328170/Put/seqid=0 2024-11-10T09:18:48,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742411_1587 (size=12001) 2024-11-10T09:18:48,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/16b73c10ec7b4b6491308d03677fa0f6 2024-11-10T09:18:48,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0a1f5f9f73d745148e09a1a5e18f656d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d 2024-11-10T09:18:48,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d, entries=200, sequenceid=86, filesize=38.6 K 2024-11-10T09:18:48,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/085bc5ca9a084498aee8211074d9f8b8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/085bc5ca9a084498aee8211074d9f8b8 2024-11-10T09:18:48,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/085bc5ca9a084498aee8211074d9f8b8, entries=150, sequenceid=86, filesize=11.7 K 2024-11-10T09:18:48,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/16b73c10ec7b4b6491308d03677fa0f6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/16b73c10ec7b4b6491308d03677fa0f6 2024-11-10T09:18:48,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/16b73c10ec7b4b6491308d03677fa0f6, entries=150, sequenceid=86, filesize=11.7 K 2024-11-10T09:18:48,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~203.05 KB/207928, currentSize=107.34 KB/109920 for 38741dfb7340c00e6e9f2028384a14b8 in 140ms, sequenceid=86, compaction requested=true 2024-11-10T09:18:48,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:48,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:48,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:18:48,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:48,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T09:18:48,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:48,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-10T09:18:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:48,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:18:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:48,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f64066b6c5d94cdb8aa856d1b9c66d47_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230328214/Put/seqid=0 2024-11-10T09:18:48,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230388338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,348 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:48,349 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/8fc19a67b6a94a5c97ca6d90f8a3a920 is 175, key is test_row_0/A:col10/1731230326978/Put/seqid=0 2024-11-10T09:18:48,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230388343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,350 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d27876400f0d4a43add757778d6fa197 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d27876400f0d4a43add757778d6fa197 2024-11-10T09:18:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230388345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230388345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,355 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into d27876400f0d4a43add757778d6fa197(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:48,355 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:48,355 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230327887; duration=0sec 2024-11-10T09:18:48,356 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-10T09:18:48,356 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:48,356 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-10T09:18:48,357 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:18:48,357 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:18:48,357 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. because compaction request was cancelled 2024-11-10T09:18:48,357 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:48,357 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:48,358 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:48,358 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:18:48,358 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:48,358 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/1923b46a014f4db79cdefcd92a8b110e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7534425065a848f3a43a61f201b9d940, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/0fe8e1830999474b902dc6c77fa89956, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/16b73c10ec7b4b6491308d03677fa0f6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=46.9 K 2024-11-10T09:18:48,358 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 1923b46a014f4db79cdefcd92a8b110e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230324424 2024-11-10T09:18:48,358 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7534425065a848f3a43a61f201b9d940, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731230324480 2024-11-10T09:18:48,359 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe8e1830999474b902dc6c77fa89956, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731230326655 2024-11-10T09:18:48,359 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b73c10ec7b4b6491308d03677fa0f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731230328162 2024-11-10T09:18:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742412_1588 (size=14594) 2024-11-10T09:18:48,367 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:48,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f64066b6c5d94cdb8aa856d1b9c66d47_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f64066b6c5d94cdb8aa856d1b9c66d47_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:48,373 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#501 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:48,373 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/083719572c0b4107aae50826ab150351 is 50, key is test_row_0/C:col10/1731230328170/Put/seqid=0 2024-11-10T09:18:48,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/e956b1665f464c30b9930d6db10da5f2, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:48,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/e956b1665f464c30b9930d6db10da5f2 is 175, key is test_row_0/A:col10/1731230328214/Put/seqid=0 2024-11-10T09:18:48,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742413_1589 (size=31058) 2024-11-10T09:18:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742415_1591 (size=12139) 2024-11-10T09:18:48,388 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/8fc19a67b6a94a5c97ca6d90f8a3a920 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8fc19a67b6a94a5c97ca6d90f8a3a920 2024-11-10T09:18:48,393 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into 8fc19a67b6a94a5c97ca6d90f8a3a920(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:48,393 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230327887; duration=0sec 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. because compaction request was cancelled 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:48,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:18:48,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742414_1590 (size=39549) 2024-11-10T09:18:48,394 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:18:48,394 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:18:48,394 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. because compaction request was cancelled 2024-11-10T09:18:48,394 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:48,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/e956b1665f464c30b9930d6db10da5f2 2024-11-10T09:18:48,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/5bdd5c3c321746f8a2186c2f9117a108 is 50, key is test_row_0/B:col10/1731230328214/Put/seqid=0 2024-11-10T09:18:48,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742416_1592 (size=12001) 2024-11-10T09:18:48,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230388457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230388457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230388457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230388457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:18:48,657 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:48,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-10T09:18:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:18:48,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:48,660 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:48,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:48,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230388667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230388667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230388667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230388667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230388687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,691 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:18:48,789 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/083719572c0b4107aae50826ab150351 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/083719572c0b4107aae50826ab150351 2024-11-10T09:18:48,792 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into 083719572c0b4107aae50826ab150351(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:48,792 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:48,792 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=12, startTime=1731230328312; duration=0sec 2024-11-10T09:18:48,792 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:48,792 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:48,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/5bdd5c3c321746f8a2186c2f9117a108 2024-11-10T09:18:48,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:18:48,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:48,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:48,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:48,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:48,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/28007b6bbce64e33b07137da620fa3bf is 50, key is test_row_0/C:col10/1731230328214/Put/seqid=0 2024-11-10T09:18:48,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:48,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742417_1593 (size=12001) 2024-11-10T09:18:48,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:18:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:48,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:18:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230388977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230388977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230388977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230388977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:18:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:49,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/28007b6bbce64e33b07137da620fa3bf 2024-11-10T09:18:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/e956b1665f464c30b9930d6db10da5f2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2 2024-11-10T09:18:49,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2, entries=200, sequenceid=106, filesize=38.6 K 2024-11-10T09:18:49,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/5bdd5c3c321746f8a2186c2f9117a108 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5bdd5c3c321746f8a2186c2f9117a108 2024-11-10T09:18:49,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5bdd5c3c321746f8a2186c2f9117a108, entries=150, sequenceid=106, filesize=11.7 K 2024-11-10T09:18:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/28007b6bbce64e33b07137da620fa3bf as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/28007b6bbce64e33b07137da620fa3bf 2024-11-10T09:18:49,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/28007b6bbce64e33b07137da620fa3bf, entries=150, sequenceid=106, filesize=11.7 K 2024-11-10T09:18:49,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 38741dfb7340c00e6e9f2028384a14b8 in 921ms, sequenceid=106, compaction requested=true 2024-11-10T09:18:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:49,250 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:49,251 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:49,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:49,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:49,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:49,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:49,252 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:49,252 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:49,252 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:18:49,252 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:18:49,253 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,253 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,253 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8fc19a67b6a94a5c97ca6d90f8a3a920, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=107.6 K 2024-11-10T09:18:49,253 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d27876400f0d4a43add757778d6fa197, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/085bc5ca9a084498aee8211074d9f8b8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5bdd5c3c321746f8a2186c2f9117a108] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=35.3 K 2024-11-10T09:18:49,253 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,253 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8fc19a67b6a94a5c97ca6d90f8a3a920, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2] 2024-11-10T09:18:49,253 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d27876400f0d4a43add757778d6fa197, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731230326655 2024-11-10T09:18:49,253 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc19a67b6a94a5c97ca6d90f8a3a920, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731230326655 2024-11-10T09:18:49,253 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 085bc5ca9a084498aee8211074d9f8b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731230328162 2024-11-10T09:18:49,254 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a1f5f9f73d745148e09a1a5e18f656d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731230327013 2024-11-10T09:18:49,254 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bdd5c3c321746f8a2186c2f9117a108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731230328207 2024-11-10T09:18:49,254 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting e956b1665f464c30b9930d6db10da5f2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731230328205 2024-11-10T09:18:49,273 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:49,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:18:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,275 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T09:18:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:49,282 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:49,282 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d00f74f59c514fae9362000753e10083 is 50, key is test_row_0/B:col10/1731230328214/Put/seqid=0 2024-11-10T09:18:49,283 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111082cf901a7cd54647ac56b5bd332c1a08_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:49,285 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111082cf901a7cd54647ac56b5bd332c1a08_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:49,285 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111082cf901a7cd54647ac56b5bd332c1a08_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:18:49,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032d52cd90bb14fe0918932f73b3b1731_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230328334/Put/seqid=0 2024-11-10T09:18:49,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742418_1594 (size=12207) 2024-11-10T09:18:49,332 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d00f74f59c514fae9362000753e10083 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d00f74f59c514fae9362000753e10083 2024-11-10T09:18:49,338 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into d00f74f59c514fae9362000753e10083(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:49,338 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230329250; duration=0sec 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:18:49,338 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. because compaction request was cancelled 2024-11-10T09:18:49,339 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:49,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742420_1596 (size=12154) 2024-11-10T09:18:49,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742419_1595 (size=4469) 2024-11-10T09:18:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:49,349 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032d52cd90bb14fe0918932f73b3b1731_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032d52cd90bb14fe0918932f73b3b1731_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:49,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/65f9ad56d55748f4b48954e12b2795c0, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:49,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/65f9ad56d55748f4b48954e12b2795c0 is 175, key is test_row_0/A:col10/1731230328334/Put/seqid=0 2024-11-10T09:18:49,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742421_1597 (size=30955) 2024-11-10T09:18:49,387 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/65f9ad56d55748f4b48954e12b2795c0 2024-11-10T09:18:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/76b8d9bc2a784fdbb0f3352329b4be95 is 50, key is test_row_0/B:col10/1731230328334/Put/seqid=0 2024-11-10T09:18:49,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742422_1598 (size=12001) 2024-11-10T09:18:49,418 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/76b8d9bc2a784fdbb0f3352329b4be95 2024-11-10T09:18:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/58882a6a88a6491d984c1d0ef56df622 is 50, key is test_row_0/C:col10/1731230328334/Put/seqid=0 2024-11-10T09:18:49,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742423_1599 (size=12001) 2024-11-10T09:18:49,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:49,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230389508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230389509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230389510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230389513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230389617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230389627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230389627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230389627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,743 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#504 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:49,744 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/19a6727c33ab41a3873e47092a1d0133 is 175, key is test_row_0/A:col10/1731230328214/Put/seqid=0 2024-11-10T09:18:49,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742424_1600 (size=31161) 2024-11-10T09:18:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:18:49,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230389827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230389837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230389837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230389837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:49,846 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/58882a6a88a6491d984c1d0ef56df622 2024-11-10T09:18:49,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/65f9ad56d55748f4b48954e12b2795c0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0 2024-11-10T09:18:49,859 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0, entries=150, sequenceid=122, filesize=30.2 K 2024-11-10T09:18:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/76b8d9bc2a784fdbb0f3352329b4be95 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/76b8d9bc2a784fdbb0f3352329b4be95 2024-11-10T09:18:49,864 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/76b8d9bc2a784fdbb0f3352329b4be95, entries=150, sequenceid=122, filesize=11.7 K 2024-11-10T09:18:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/58882a6a88a6491d984c1d0ef56df622 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/58882a6a88a6491d984c1d0ef56df622 2024-11-10T09:18:49,868 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/58882a6a88a6491d984c1d0ef56df622, entries=150, sequenceid=122, filesize=11.7 K 2024-11-10T09:18:49,868 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 38741dfb7340c00e6e9f2028384a14b8 in 593ms, sequenceid=122, compaction requested=true 2024-11-10T09:18:49,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:49,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:49,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-10T09:18:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-10T09:18:49,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T09:18:49,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2090 sec 2024-11-10T09:18:49,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.2180 sec 2024-11-10T09:18:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:50,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T09:18:50,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:50,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:50,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:50,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:50,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:50,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:50,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d7434a6c7aae4d2e85d77bb0f13a53bb_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:50,156 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/19a6727c33ab41a3873e47092a1d0133 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/19a6727c33ab41a3873e47092a1d0133 2024-11-10T09:18:50,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742425_1601 (size=14794) 2024-11-10T09:18:50,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:50,161 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into 19a6727c33ab41a3873e47092a1d0133(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:50,161 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:50,161 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230329250; duration=0sec 2024-11-10T09:18:50,161 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:50,161 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:50,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230390157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230390158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230390159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d7434a6c7aae4d2e85d77bb0f13a53bb_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7434a6c7aae4d2e85d77bb0f13a53bb_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230390160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/405316df1eb1414294861d13fc2c5a90, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:50,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/405316df1eb1414294861d13fc2c5a90 is 175, key is test_row_0/A:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:50,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742426_1602 (size=39749) 2024-11-10T09:18:50,194 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/405316df1eb1414294861d13fc2c5a90 2024-11-10T09:18:50,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/df43d80d3bd748e89e0846ff900c5687 is 50, key is test_row_0/B:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:50,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742427_1603 (size=12151) 2024-11-10T09:18:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230390267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230390268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230390268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230390277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230390477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230390477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230390478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230390487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/df43d80d3bd748e89e0846ff900c5687 2024-11-10T09:18:50,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/e868c09327154d419a046b26bb94bc06 is 50, key is test_row_0/C:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:50,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742428_1604 (size=12151) 2024-11-10T09:18:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230390788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230390788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230390788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230390798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:18:50,807 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:50,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-10T09:18:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:18:50,810 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:50,811 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:50,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:18:50,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:18:50,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:50,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:50,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:50,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:50,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:51,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/e868c09327154d419a046b26bb94bc06 2024-11-10T09:18:51,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:18:51,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:51,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:51,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:51,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/405316df1eb1414294861d13fc2c5a90 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90 2024-11-10T09:18:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:18:51,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90, entries=200, sequenceid=143, filesize=38.8 K 2024-11-10T09:18:51,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/df43d80d3bd748e89e0846ff900c5687 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/df43d80d3bd748e89e0846ff900c5687 2024-11-10T09:18:51,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/df43d80d3bd748e89e0846ff900c5687, entries=150, sequenceid=143, filesize=11.9 K 2024-11-10T09:18:51,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/e868c09327154d419a046b26bb94bc06 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/e868c09327154d419a046b26bb94bc06 2024-11-10T09:18:51,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/e868c09327154d419a046b26bb94bc06, entries=150, sequenceid=143, filesize=11.9 K 2024-11-10T09:18:51,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 38741dfb7340c00e6e9f2028384a14b8 in 1010ms, sequenceid=143, compaction requested=true 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:51,151 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:51,151 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:51,152 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:51,152 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:18:51,152 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,152 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d00f74f59c514fae9362000753e10083, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/76b8d9bc2a784fdbb0f3352329b4be95, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/df43d80d3bd748e89e0846ff900c5687] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=35.5 K 2024-11-10T09:18:51,152 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:51,152 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:18:51,152 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,152 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/19a6727c33ab41a3873e47092a1d0133, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=99.5 K 2024-11-10T09:18:51,152 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,152 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/19a6727c33ab41a3873e47092a1d0133, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90] 2024-11-10T09:18:51,153 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a6727c33ab41a3873e47092a1d0133, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731230328207 2024-11-10T09:18:51,153 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f9ad56d55748f4b48954e12b2795c0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731230328334 2024-11-10T09:18:51,153 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting d00f74f59c514fae9362000753e10083, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731230328207 2024-11-10T09:18:51,153 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 405316df1eb1414294861d13fc2c5a90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731230329506 2024-11-10T09:18:51,153 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b8d9bc2a784fdbb0f3352329b4be95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731230328334 2024-11-10T09:18:51,153 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting df43d80d3bd748e89e0846ff900c5687, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731230329506 2024-11-10T09:18:51,161 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:51,161 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/627529f55343452ca516fc15b00e615c is 50, key is test_row_0/B:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:51,165 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:51,172 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110cf6fe8111c9e43e28566e6e12b84d343_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:51,174 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110cf6fe8111c9e43e28566e6e12b84d343_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:51,174 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cf6fe8111c9e43e28566e6e12b84d343_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:51,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742429_1605 (size=12459) 2024-11-10T09:18:51,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742430_1606 (size=4469) 2024-11-10T09:18:51,181 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#513 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:51,181 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/7182a94c639a4da9baa9830e2083007c is 175, key is test_row_0/A:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:51,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742431_1607 (size=31413) 2024-11-10T09:18:51,191 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/7182a94c639a4da9baa9830e2083007c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/7182a94c639a4da9baa9830e2083007c 2024-11-10T09:18:51,195 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into 7182a94c639a4da9baa9830e2083007c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:51,195 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:51,195 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230331151; duration=0sec 2024-11-10T09:18:51,195 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:51,195 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:51,195 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:18:51,196 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:18:51,196 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:18:51,196 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,196 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/083719572c0b4107aae50826ab150351, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/28007b6bbce64e33b07137da620fa3bf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/58882a6a88a6491d984c1d0ef56df622, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/e868c09327154d419a046b26bb94bc06] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=47.2 K 2024-11-10T09:18:51,197 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 083719572c0b4107aae50826ab150351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731230328162 2024-11-10T09:18:51,197 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 28007b6bbce64e33b07137da620fa3bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731230328207 2024-11-10T09:18:51,197 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 58882a6a88a6491d984c1d0ef56df622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731230328334 2024-11-10T09:18:51,198 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e868c09327154d419a046b26bb94bc06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731230329506 2024-11-10T09:18:51,206 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:51,207 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/794bb24332de4894a7a975eb792fe994 is 50, key is test_row_0/C:col10/1731230329506/Put/seqid=0 2024-11-10T09:18:51,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742432_1608 (size=12425) 2024-11-10T09:18:51,238 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/794bb24332de4894a7a975eb792fe994 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/794bb24332de4894a7a975eb792fe994 2024-11-10T09:18:51,242 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into 794bb24332de4894a7a975eb792fe994(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:51,242 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:51,242 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=12, startTime=1731230331151; duration=0sec 2024-11-10T09:18:51,242 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:51,242 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:51,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:18:51,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:51,271 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T09:18:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8d5ac34e7634c479dd90394a2f63778_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230330157/Put/seqid=0 2024-11-10T09:18:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742433_1609 (size=12304) 2024-11-10T09:18:51,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731230331307, max timestamp=1731230331328Num uniques 859; 2024-11-10T09:18:51,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T09:18:51,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731230331307, max timestamp=1731230331328Num uniques 859; 2024-11-10T09:18:51,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T09:18:51,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731230331307, max timestamp=1731230331328Num uniques 859; 2024-11-10T09:18:51,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T09:18:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230391357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230391360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230391360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230391362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:18:51,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230391467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230391478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230391478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230391478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,581 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/627529f55343452ca516fc15b00e615c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/627529f55343452ca516fc15b00e615c 2024-11-10T09:18:51,584 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into 627529f55343452ca516fc15b00e615c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:51,584 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:51,584 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230331151; duration=0sec 2024-11-10T09:18:51,584 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:51,584 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:51,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230391677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230391687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230391688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230391688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:51,705 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8d5ac34e7634c479dd90394a2f63778_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8d5ac34e7634c479dd90394a2f63778_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1db9429e8430431eb0c6552269288103, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:51,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1db9429e8430431eb0c6552269288103 is 175, key is test_row_0/A:col10/1731230330157/Put/seqid=0 2024-11-10T09:18:51,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742434_1610 (size=31105) 2024-11-10T09:18:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:18:51,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230391987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230391997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230391997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:51,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230392007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,118 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1db9429e8430431eb0c6552269288103 2024-11-10T09:18:52,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/03977c60f9cd49c08901a2777f91fa63 is 50, key is test_row_0/B:col10/1731230330157/Put/seqid=0 2024-11-10T09:18:52,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742435_1611 (size=12151) 2024-11-10T09:18:52,185 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/03977c60f9cd49c08901a2777f91fa63 2024-11-10T09:18:52,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/7c5cbd13e3e048a18250f4e4fa3797e2 is 50, key is test_row_0/C:col10/1731230330157/Put/seqid=0 2024-11-10T09:18:52,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742436_1612 (size=12151) 2024-11-10T09:18:52,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230392497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230392507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230392507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230392517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,640 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/7c5cbd13e3e048a18250f4e4fa3797e2 2024-11-10T09:18:52,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/1db9429e8430431eb0c6552269288103 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103 2024-11-10T09:18:52,648 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103, entries=150, sequenceid=158, filesize=30.4 K 2024-11-10T09:18:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/03977c60f9cd49c08901a2777f91fa63 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/03977c60f9cd49c08901a2777f91fa63 2024-11-10T09:18:52,654 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/03977c60f9cd49c08901a2777f91fa63, entries=150, sequenceid=158, filesize=11.9 K 2024-11-10T09:18:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/7c5cbd13e3e048a18250f4e4fa3797e2 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7c5cbd13e3e048a18250f4e4fa3797e2 2024-11-10T09:18:52,663 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7c5cbd13e3e048a18250f4e4fa3797e2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-10T09:18:52,664 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 38741dfb7340c00e6e9f2028384a14b8 in 1393ms, sequenceid=158, compaction requested=false 2024-11-10T09:18:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-10T09:18:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-10T09:18:52,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T09:18:52,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8540 sec 2024-11-10T09:18:52,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8600 sec 2024-11-10T09:18:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:52,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-10T09:18:52,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:52,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:52,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:52,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:52,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:52,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:52,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107397e1dbdc2245b5a561ee8acc465b8a_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:52,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742437_1613 (size=14794) 2024-11-10T09:18:52,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:52,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107397e1dbdc2245b5a561ee8acc465b8a_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107397e1dbdc2245b5a561ee8acc465b8a_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:52,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/336627bebb50427da23d9386ea59c373, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:52,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/336627bebb50427da23d9386ea59c373 is 175, key is test_row_0/A:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742438_1614 (size=39749) 2024-11-10T09:18:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230392777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230392887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:18:52,967 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:52,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-10T09:18:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:52,970 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:52,970 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:52,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:53,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230393097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:53,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:18:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:53,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/336627bebb50427da23d9386ea59c373 2024-11-10T09:18:53,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/66d68a9aff884886954e5eb6c224d488 is 50, key is test_row_0/B:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:53,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742439_1615 (size=12151) 2024-11-10T09:18:53,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/66d68a9aff884886954e5eb6c224d488 2024-11-10T09:18:53,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/81c62e573b924ae3ac589248a737a875 is 50, key is test_row_0/C:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:53,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742440_1616 (size=12151) 2024-11-10T09:18:53,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:18:53,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:53,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:53,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230393408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:53,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:18:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230393517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:53,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230393517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230393517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:53,520 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:53,521 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:53,521 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:53,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230393539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:53,542 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_1' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:53,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:18:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:53,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/81c62e573b924ae3ac589248a737a875 2024-11-10T09:18:53,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/336627bebb50427da23d9386ea59c373 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373 2024-11-10T09:18:53,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373, entries=200, sequenceid=193, filesize=38.8 K 2024-11-10T09:18:53,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/66d68a9aff884886954e5eb6c224d488 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/66d68a9aff884886954e5eb6c224d488 2024-11-10T09:18:53,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/66d68a9aff884886954e5eb6c224d488, entries=150, sequenceid=193, filesize=11.9 K 2024-11-10T09:18:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/81c62e573b924ae3ac589248a737a875 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/81c62e573b924ae3ac589248a737a875 2024-11-10T09:18:53,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/81c62e573b924ae3ac589248a737a875, entries=150, sequenceid=193, filesize=11.9 K 2024-11-10T09:18:53,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 38741dfb7340c00e6e9f2028384a14b8 in 882ms, sequenceid=193, compaction requested=true 2024-11-10T09:18:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:53,614 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:53,614 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:53,614 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:53,614 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:18:53,614 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,614 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/7182a94c639a4da9baa9830e2083007c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=99.9 K 2024-11-10T09:18:53,614 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,614 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/7182a94c639a4da9baa9830e2083007c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373] 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:18:53,615 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,615 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/627529f55343452ca516fc15b00e615c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/03977c60f9cd49c08901a2777f91fa63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/66d68a9aff884886954e5eb6c224d488] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=35.9 K 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7182a94c639a4da9baa9830e2083007c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731230329506 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 627529f55343452ca516fc15b00e615c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731230329506 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db9429e8430431eb0c6552269288103, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731230330149 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 03977c60f9cd49c08901a2777f91fa63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731230330149 2024-11-10T09:18:53,615 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336627bebb50427da23d9386ea59c373, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731230331349 2024-11-10T09:18:53,616 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d68a9aff884886954e5eb6c224d488, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731230331349 2024-11-10T09:18:53,619 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:53,621 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:53,621 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/b4039eac32b741bc95910b3cc8bb8b38 is 50, key is test_row_0/B:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:53,623 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e52a7d2b6d7541d88c56091feb12ced8_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:53,624 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e52a7d2b6d7541d88c56091feb12ced8_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:53,625 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e52a7d2b6d7541d88c56091feb12ced8_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:53,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742441_1617 (size=12561) 2024-11-10T09:18:53,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742442_1618 (size=4469) 2024-11-10T09:18:53,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:53,735 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b6e704baba9f4536b249f7479a06ac12_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230332775/Put/seqid=0 2024-11-10T09:18:53,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742443_1619 (size=12304) 2024-11-10T09:18:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230333988, max timestamp=1731230334011Num uniques 859; 2024-11-10T09:18:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230333988, max timestamp=1731230334011Num uniques 859; 2024-11-10T09:18:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230333988, max timestamp=1731230334011Num uniques 859; 2024-11-10T09:18:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:54,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:54,029 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:54,029 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/bc7c6bd62a964b02a3ee53367912bd20 is 175, key is test_row_0/A:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:54,031 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/b4039eac32b741bc95910b3cc8bb8b38 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/b4039eac32b741bc95910b3cc8bb8b38 2024-11-10T09:18:54,039 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into b4039eac32b741bc95910b3cc8bb8b38(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:54,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:54,039 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230333614; duration=0sec 2024-11-10T09:18:54,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:54,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:54,039 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:54,040 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:54,040 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:18:54,040 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:54,041 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/794bb24332de4894a7a975eb792fe994, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7c5cbd13e3e048a18250f4e4fa3797e2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/81c62e573b924ae3ac589248a737a875] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=35.9 K 2024-11-10T09:18:54,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 794bb24332de4894a7a975eb792fe994, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731230329506 2024-11-10T09:18:54,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c5cbd13e3e048a18250f4e4fa3797e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731230330149 2024-11-10T09:18:54,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c62e573b924ae3ac589248a737a875, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731230331349 2024-11-10T09:18:54,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742444_1620 (size=31515) 2024-11-10T09:18:54,050 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:54,051 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/42534e5711874dcba08eb50a486856cb is 50, key is test_row_0/C:col10/1731230332729/Put/seqid=0 2024-11-10T09:18:54,053 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/bc7c6bd62a964b02a3ee53367912bd20 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/bc7c6bd62a964b02a3ee53367912bd20 2024-11-10T09:18:54,063 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into bc7c6bd62a964b02a3ee53367912bd20(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:54,063 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:54,063 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230333614; duration=0sec 2024-11-10T09:18:54,063 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:54,063 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:54,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742445_1621 (size=12527) 2024-11-10T09:18:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:54,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230394116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:54,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:54,192 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b6e704baba9f4536b249f7479a06ac12_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b6e704baba9f4536b249f7479a06ac12_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3e73ef7f6d464559a7e8a5a6bed7d83b, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3e73ef7f6d464559a7e8a5a6bed7d83b is 175, key is test_row_0/A:col10/1731230332775/Put/seqid=0 2024-11-10T09:18:54,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742446_1622 (size=31105) 2024-11-10T09:18:54,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230394227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230394437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:54,484 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/42534e5711874dcba08eb50a486856cb as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/42534e5711874dcba08eb50a486856cb 2024-11-10T09:18:54,487 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into 42534e5711874dcba08eb50a486856cb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:54,487 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:54,487 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=13, startTime=1731230333614; duration=0sec 2024-11-10T09:18:54,487 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:54,487 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:54,614 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3e73ef7f6d464559a7e8a5a6bed7d83b 2024-11-10T09:18:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/2cd8a8b2e6184a1baa1902b146efdcf1 is 50, key is test_row_0/B:col10/1731230332775/Put/seqid=0 2024-11-10T09:18:54,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742447_1623 (size=12151) 2024-11-10T09:18:54,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230394747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:55,030 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/2cd8a8b2e6184a1baa1902b146efdcf1 2024-11-10T09:18:55,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/75b50b44ef5944b181d42cff984de012 is 50, key is test_row_0/C:col10/1731230332775/Put/seqid=0 2024-11-10T09:18:55,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742448_1624 (size=12151) 2024-11-10T09:18:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230395260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:55,445 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/75b50b44ef5944b181d42cff984de012 2024-11-10T09:18:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3e73ef7f6d464559a7e8a5a6bed7d83b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b 2024-11-10T09:18:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b, entries=150, sequenceid=204, filesize=30.4 K 2024-11-10T09:18:55,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/2cd8a8b2e6184a1baa1902b146efdcf1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/2cd8a8b2e6184a1baa1902b146efdcf1 2024-11-10T09:18:55,460 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/2cd8a8b2e6184a1baa1902b146efdcf1, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T09:18:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/75b50b44ef5944b181d42cff984de012 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/75b50b44ef5944b181d42cff984de012 2024-11-10T09:18:55,463 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/75b50b44ef5944b181d42cff984de012, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T09:18:55,464 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 38741dfb7340c00e6e9f2028384a14b8 in 1729ms, sequenceid=204, compaction requested=false 2024-11-10T09:18:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-10T09:18:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-10T09:18:55,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T09:18:55,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4960 sec 2024-11-10T09:18:55,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.5000 sec 2024-11-10T09:18:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:55,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T09:18:55,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230334064, max timestamp=1731230335542Num uniques 1018; 2024-11-10T09:18:55,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230334064, max timestamp=1731230335542Num uniques 1018; 2024-11-10T09:18:55,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T09:18:55,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T09:18:55,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230334064, max timestamp=1731230335542Num uniques 1018; 2024-11-10T09:18:55,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T09:18:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:55,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108912d62c5de24e13b6129218b8ac40b9_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:55,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731230335579, max timestamp=1731230335591Num uniques 859; 2024-11-10T09:18:55,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230335579, max timestamp=1731230335591Num uniques 859; 2024-11-10T09:18:55,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:55,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:55,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230335579, max timestamp=1731230335591Num uniques 859; 2024-11-10T09:18:55,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:55,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742449_1625 (size=14794) 2024-11-10T09:18:55,616 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:55,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108912d62c5de24e13b6129218b8ac40b9_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108912d62c5de24e13b6129218b8ac40b9_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:55,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c51636c4c62640f19a19b2740a1ba4b6, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:55,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c51636c4c62640f19a19b2740a1ba4b6 is 175, key is test_row_0/A:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230335595, max timestamp=1731230335644Num uniques 1068; 2024-11-10T09:18:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T09:18:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230335595, max timestamp=1731230335644Num uniques 1018; 2024-11-10T09:18:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T09:18:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731230335595, max timestamp=1731230335644Num uniques 1018; 2024-11-10T09:18:55,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T09:18:55,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742450_1626 (size=39749) 2024-11-10T09:18:55,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c51636c4c62640f19a19b2740a1ba4b6 2024-11-10T09:18:55,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/6a997519e93e41cd8a32e0f8b33805de is 50, key is test_row_0/B:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:55,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742451_1627 (size=12151) 2024-11-10T09:18:55,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/6a997519e93e41cd8a32e0f8b33805de 2024-11-10T09:18:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731230335663, max timestamp=1731230335685Num uniques 1077; 2024-11-10T09:18:55,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-10T09:18:55,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8100, totalHeapSize=18648, min timestamp=1731230335675, max timestamp=1731230335688Num uniques 1077; 2024-11-10T09:18:55,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230335675, max timestamp=1731230335688Num uniques 1077; 2024-11-10T09:18:55,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49320, new segment data size=8100, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T09:18:55,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T09:18:55,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/dca880b578f148abb952666c725d413e is 50, key is test_row_0/C:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742452_1628 (size=12151) 2024-11-10T09:18:55,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/dca880b578f148abb952666c725d413e 2024-11-10T09:18:55,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c51636c4c62640f19a19b2740a1ba4b6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6 2024-11-10T09:18:55,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6, entries=200, sequenceid=245, filesize=38.8 K 2024-11-10T09:18:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/6a997519e93e41cd8a32e0f8b33805de as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6a997519e93e41cd8a32e0f8b33805de 2024-11-10T09:18:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6a997519e93e41cd8a32e0f8b33805de, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T09:18:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/dca880b578f148abb952666c725d413e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dca880b578f148abb952666c725d413e 2024-11-10T09:18:55,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dca880b578f148abb952666c725d413e, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T09:18:55,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=131.94 KB/135110 for 38741dfb7340c00e6e9f2028384a14b8 in 252ms, sequenceid=245, compaction requested=true 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:55,795 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:55,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:18:55,795 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:55,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=145.36 KB heapSize=414.08 KB 2024-11-10T09:18:55,796 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:55,796 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:18:55,796 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:55,797 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/bc7c6bd62a964b02a3ee53367912bd20, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=100.0 K 2024-11-10T09:18:55,797 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:55,797 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/bc7c6bd62a964b02a3ee53367912bd20, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6] 2024-11-10T09:18:55,798 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:55,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1731230335729, max timestamp=1731230335767Num uniques 1136; 2024-11-10T09:18:55,798 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc7c6bd62a964b02a3ee53367912bd20, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731230331349 2024-11-10T09:18:55,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52016, new segment data size=10796, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-10T09:18:55,798 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:18:55,798 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:55,798 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/b4039eac32b741bc95910b3cc8bb8b38, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/2cd8a8b2e6184a1baa1902b146efdcf1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6a997519e93e41cd8a32e0f8b33805de] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.0 K 2024-11-10T09:18:55,798 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e73ef7f6d464559a7e8a5a6bed7d83b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731230332763 2024-11-10T09:18:55,798 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting b4039eac32b741bc95910b3cc8bb8b38, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731230331349 2024-11-10T09:18:55,798 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51636c4c62640f19a19b2740a1ba4b6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731230334064 2024-11-10T09:18:55,799 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd8a8b2e6184a1baa1902b146efdcf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731230332763 2024-11-10T09:18:55,799 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a997519e93e41cd8a32e0f8b33805de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731230334114 2024-11-10T09:18:55,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230335729, max timestamp=1731230335767Num uniques 1036; 2024-11-10T09:18:55,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-10T09:18:55,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:55,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230335729, max timestamp=1731230335767Num uniques 1036; 2024-11-10T09:18:55,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-10T09:18:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:55,809 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:55,825 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#531 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:55,827 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e798bb4dbc6e425eaef15192a576c644 is 50, key is test_row_0/B:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:55,838 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111017b2dfdd807741f6a59431e9dddcd094_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:55,839 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111017b2dfdd807741f6a59431e9dddcd094_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:55,840 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111017b2dfdd807741f6a59431e9dddcd094_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:55,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230335835, max timestamp=1731230335849Num uniques 859; 2024-11-10T09:18:55,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230335835, max timestamp=1731230335849Num uniques 859; 2024-11-10T09:18:55,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:55,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:55,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731230335835, max timestamp=1731230335849Num uniques 859; 2024-11-10T09:18:55,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T09:18:55,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111031238bfd41294b2196f778d75cbdb8b6_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230335795/Put/seqid=0 2024-11-10T09:18:55,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742453_1629 (size=12663) 2024-11-10T09:18:55,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742454_1630 (size=4469) 2024-11-10T09:18:55,878 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e798bb4dbc6e425eaef15192a576c644 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e798bb4dbc6e425eaef15192a576c644 2024-11-10T09:18:55,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742455_1631 (size=14994) 2024-11-10T09:18:55,882 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into e798bb4dbc6e425eaef15192a576c644(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:55,882 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:55,882 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230335795; duration=0sec 2024-11-10T09:18:55,883 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:55,883 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:55,883 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:55,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:55,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:55,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:18:55,885 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:55,885 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/42534e5711874dcba08eb50a486856cb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/75b50b44ef5944b181d42cff984de012, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dca880b578f148abb952666c725d413e] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.0 K 2024-11-10T09:18:55,886 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 42534e5711874dcba08eb50a486856cb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731230331349 2024-11-10T09:18:55,887 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b50b44ef5944b181d42cff984de012, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731230332763 2024-11-10T09:18:55,887 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting dca880b578f148abb952666c725d413e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731230334114 2024-11-10T09:18:55,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111031238bfd41294b2196f778d75cbdb8b6_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111031238bfd41294b2196f778d75cbdb8b6_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:55,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/6272cfdf2d5d414a8a89516d56af6285, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:55,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/6272cfdf2d5d414a8a89516d56af6285 is 175, key is test_row_0/A:col10/1731230335795/Put/seqid=0 2024-11-10T09:18:55,898 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12262, totalHeapSize=28112, min timestamp=1731230335872, max timestamp=1731230335894Num uniques 1068; 2024-11-10T09:18:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48902, new segment data size=12262, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-10T09:18:55,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731230335872, max timestamp=1731230335894Num uniques 1068; 2024-11-10T09:18:55,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-10T09:18:55,899 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/eb365a65e53e47498e686f4ebe54171d is 50, key is test_row_0/C:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:55,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7685, totalHeapSize=17712, min timestamp=1731230335881, max timestamp=1731230335894Num uniques 1068; 2024-11-10T09:18:55,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48905, new segment data size=7685, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-10T09:18:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742456_1632 (size=39949) 2024-11-10T09:18:55,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742457_1633 (size=12629) 2024-11-10T09:18:55,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230395941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:55,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731230335935, max timestamp=1731230335946Num uniques 1127; 2024-11-10T09:18:55,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-10T09:18:55,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731230335935, max timestamp=1731230335946Num uniques 1127; 2024-11-10T09:18:55,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-10T09:18:55,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731230335935, max timestamp=1731230335946Num uniques 1027; 2024-11-10T09:18:55,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-10T09:18:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:55,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230395986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:55,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230395988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:55,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230395989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230396057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230396097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230396097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230396097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,270 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#530 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:56,270 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c34eda6003804439901e27bc64152634 is 175, key is test_row_0/A:col10/1731230334114/Put/seqid=0 2024-11-10T09:18:56,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230396267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742458_1634 (size=31617) 2024-11-10T09:18:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230396277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,282 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:18:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230396307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/6272cfdf2d5d414a8a89516d56af6285 2024-11-10T09:18:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230396307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230396307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/100743fbc25142c68c57a6d2e6d4a8d7 is 50, key is test_row_0/B:col10/1731230335795/Put/seqid=0 2024-11-10T09:18:56,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742459_1635 (size=12301) 2024-11-10T09:18:56,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.82 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/100743fbc25142c68c57a6d2e6d4a8d7 2024-11-10T09:18:56,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/a1cdcae567de497b8b37d992241aace7 is 50, key is test_row_0/C:col10/1731230335795/Put/seqid=0 2024-11-10T09:18:56,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742460_1636 (size=12301) 2024-11-10T09:18:56,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.82 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/a1cdcae567de497b8b37d992241aace7 2024-11-10T09:18:56,346 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/eb365a65e53e47498e686f4ebe54171d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/eb365a65e53e47498e686f4ebe54171d 2024-11-10T09:18:56,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/6272cfdf2d5d414a8a89516d56af6285 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285 2024-11-10T09:18:56,351 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into eb365a65e53e47498e686f4ebe54171d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:56,351 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:56,351 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=13, startTime=1731230335795; duration=0sec 2024-11-10T09:18:56,351 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:56,351 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:56,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285, entries=200, sequenceid=319, filesize=39.0 K 2024-11-10T09:18:56,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/100743fbc25142c68c57a6d2e6d4a8d7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/100743fbc25142c68c57a6d2e6d4a8d7 2024-11-10T09:18:56,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/100743fbc25142c68c57a6d2e6d4a8d7, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T09:18:56,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/a1cdcae567de497b8b37d992241aace7 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/a1cdcae567de497b8b37d992241aace7 2024-11-10T09:18:56,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/a1cdcae567de497b8b37d992241aace7, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T09:18:56,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.29 KB/40238, heapSize ~101.27 KB/103696, currentSize=147.60 KB/151140 for 38741dfb7340c00e6e9f2028384a14b8 in 592ms, sequenceid=319, compaction requested=false 2024-11-10T09:18:56,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:56,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-11-10T09:18:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:18:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:18:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:18:56,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c16bee5e1c3143c6ba75d059886eab1f_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:56,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742461_1637 (size=12454) 2024-11-10T09:18:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230396611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230396617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230396617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230396617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,680 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c34eda6003804439901e27bc64152634 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c34eda6003804439901e27bc64152634 2024-11-10T09:18:56,683 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into c34eda6003804439901e27bc64152634(size=30.9 K), total size for store is 69.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:56,683 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:56,684 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230335795; duration=0sec 2024-11-10T09:18:56,684 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:56,684 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:56,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230396717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230396930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:57,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:57,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c16bee5e1c3143c6ba75d059886eab1f_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c16bee5e1c3143c6ba75d059886eab1f_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:57,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/ef34087d78324afc8530cb7156e37cf6, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:57,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/ef34087d78324afc8530cb7156e37cf6 is 175, key is test_row_0/A:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:57,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742462_1638 (size=31255) 2024-11-10T09:18:57,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/ef34087d78324afc8530cb7156e37cf6 2024-11-10T09:18:57,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d0afdf12ed6e40508842f6c47a8058c1 is 50, key is test_row_0/B:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:57,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742463_1639 (size=12301) 2024-11-10T09:18:57,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d0afdf12ed6e40508842f6c47a8058c1 2024-11-10T09:18:57,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/73b45f0f9cb14353b4f2c739269e4d77 is 50, key is test_row_0/C:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:57,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230397128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:57,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230397128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:57,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742464_1640 (size=12301) 2024-11-10T09:18:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:18:57,137 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:57,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T09:18:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:18:57,139 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:57,139 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:57,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230397137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:18:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230397247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:57,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:18:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:57,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:18:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:18:57,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/73b45f0f9cb14353b4f2c739269e4d77 2024-11-10T09:18:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/ef34087d78324afc8530cb7156e37cf6 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6 2024-11-10T09:18:57,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6, entries=150, sequenceid=393, filesize=30.5 K 2024-11-10T09:18:57,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d0afdf12ed6e40508842f6c47a8058c1 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d0afdf12ed6e40508842f6c47a8058c1 2024-11-10T09:18:57,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d0afdf12ed6e40508842f6c47a8058c1, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T09:18:57,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/73b45f0f9cb14353b4f2c739269e4d77 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/73b45f0f9cb14353b4f2c739269e4d77 2024-11-10T09:18:57,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/73b45f0f9cb14353b4f2c739269e4d77, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T09:18:57,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=26.84 KB/27480 for 38741dfb7340c00e6e9f2028384a14b8 in 975ms, sequenceid=393, compaction requested=true 2024-11-10T09:18:57,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:57,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:18:57,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:57,556 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:57,556 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:57,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:18:57,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:57,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:18:57,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:57,557 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:57,557 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:18:57,557 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,558 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e798bb4dbc6e425eaef15192a576c644, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/100743fbc25142c68c57a6d2e6d4a8d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d0afdf12ed6e40508842f6c47a8058c1] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.4 K 2024-11-10T09:18:57,558 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:57,558 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:18:57,558 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,558 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c34eda6003804439901e27bc64152634, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=100.4 K 2024-11-10T09:18:57,558 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,558 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c34eda6003804439901e27bc64152634, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6] 2024-11-10T09:18:57,559 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting c34eda6003804439901e27bc64152634, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731230334114 2024-11-10T09:18:57,559 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e798bb4dbc6e425eaef15192a576c644, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731230334114 2024-11-10T09:18:57,559 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 100743fbc25142c68c57a6d2e6d4a8d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230335729 2024-11-10T09:18:57,559 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6272cfdf2d5d414a8a89516d56af6285, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230335729 2024-11-10T09:18:57,563 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef34087d78324afc8530cb7156e37cf6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230335980 2024-11-10T09:18:57,563 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d0afdf12ed6e40508842f6c47a8058c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230335980 2024-11-10T09:18:57,567 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:57,569 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:57,570 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/5525b6434b3d433093aac88692d404b0 is 50, key is test_row_0/B:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:57,573 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100cc715369c0947d9af7e8c7e9aefdfae_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:57,575 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100cc715369c0947d9af7e8c7e9aefdfae_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:57,575 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100cc715369c0947d9af7e8c7e9aefdfae_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:57,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742466_1642 (size=4469) 2024-11-10T09:18:57,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742465_1641 (size=12915) 2024-11-10T09:18:57,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:18:57,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,599 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T09:18:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:18:57,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110664b1a58e16e4859948e9c362318fb6f_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230336605/Put/seqid=0 2024-11-10T09:18:57,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742467_1643 (size=12454) 2024-11-10T09:18:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:18:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230337851, max timestamp=1731230337878Num uniques 859; 2024-11-10T09:18:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:57,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230337851, max timestamp=1731230337878Num uniques 859; 2024-11-10T09:18:57,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230337851, max timestamp=1731230337878Num uniques 859; 2024-11-10T09:18:57,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T09:18:57,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:18:57,983 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#539 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:57,984 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/516035b0768f4458b9eaba445fded3ee is 175, key is test_row_0/A:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:57,989 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/5525b6434b3d433093aac88692d404b0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5525b6434b3d433093aac88692d404b0 2024-11-10T09:18:57,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742468_1644 (size=31869) 2024-11-10T09:18:57,994 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into 5525b6434b3d433093aac88692d404b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:57,994 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:57,994 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230337556; duration=0sec 2024-11-10T09:18:57,994 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:18:57,994 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:18:57,994 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:18:57,995 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:18:57,995 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:18:57,995 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:57,995 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/eb365a65e53e47498e686f4ebe54171d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/a1cdcae567de497b8b37d992241aace7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/73b45f0f9cb14353b4f2c739269e4d77] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.4 K 2024-11-10T09:18:57,995 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting eb365a65e53e47498e686f4ebe54171d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731230334114 2024-11-10T09:18:57,995 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting a1cdcae567de497b8b37d992241aace7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230335729 2024-11-10T09:18:57,996 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b45f0f9cb14353b4f2c739269e4d77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230335980 2024-11-10T09:18:57,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731230337960, max timestamp=1731230337996Num uniques 1018; 2024-11-10T09:18:57,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731230337960, max timestamp=1731230337996Num uniques 1018; 2024-11-10T09:18:58,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T09:18:58,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T09:18:58,002 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:18:58,002 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/437b54178f4e4d2eba39ffe62765f67c is 50, key is test_row_0/C:col10/1731230335980/Put/seqid=0 2024-11-10T09:18:58,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230337960, max timestamp=1731230337996Num uniques 1009; 2024-11-10T09:18:58,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-10T09:18:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:58,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742469_1645 (size=12881) 2024-11-10T09:18:58,013 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110664b1a58e16e4859948e9c362318fb6f_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110664b1a58e16e4859948e9c362318fb6f_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/96e1ec9ee0484ad9bc2884ca13169d15, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/96e1ec9ee0484ad9bc2884ca13169d15 is 175, key is test_row_0/A:col10/1731230336605/Put/seqid=0 2024-11-10T09:18:58,020 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/437b54178f4e4d2eba39ffe62765f67c as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/437b54178f4e4d2eba39ffe62765f67c 2024-11-10T09:18:58,025 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into 437b54178f4e4d2eba39ffe62765f67c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:58,025 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:58,025 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=13, startTime=1731230337557; duration=0sec 2024-11-10T09:18:58,025 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:58,026 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:18:58,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742470_1646 (size=31255) 2024-11-10T09:18:58,030 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/96e1ec9ee0484ad9bc2884ca13169d15 2024-11-10T09:18:58,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d8cb461a4f70416aa3964fc8e756bdd8 is 50, key is test_row_0/B:col10/1731230336605/Put/seqid=0 2024-11-10T09:18:58,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742471_1647 (size=12301) 2024-11-10T09:18:58,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230338044, max timestamp=1731230338100Num uniques 1027; 2024-11-10T09:18:58,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230338044, max timestamp=1731230338100Num uniques 1009; 2024-11-10T09:18:58,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T09:18:58,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-10T09:18:58,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731230338044, max timestamp=1731230338100Num uniques 1027; 2024-11-10T09:18:58,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T09:18:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731230338123, max timestamp=1731230338158Num uniques 1036; 2024-11-10T09:18:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-10T09:18:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731230338123, max timestamp=1731230338158Num uniques 1036; 2024-11-10T09:18:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731230338123, max timestamp=1731230338158Num uniques 1036; 2024-11-10T09:18:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-10T09:18:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-10T09:18:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731230338176, max timestamp=1731230338183Num uniques 1145; 2024-11-10T09:18:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-10T09:18:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731230338176, max timestamp=1731230338183Num uniques 1145; 2024-11-10T09:18:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-10T09:18:58,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731230338176, max timestamp=1731230338183Num uniques 1145; 2024-11-10T09:18:58,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-10T09:18:58,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731230338201, max timestamp=1731230338217Num uniques 1054; 2024-11-10T09:18:58,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-10T09:18:58,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731230338201, max timestamp=1731230338217Num uniques 1054; 2024-11-10T09:18:58,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-10T09:18:58,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731230338201, max timestamp=1731230338217Num uniques 1054; 2024-11-10T09:18:58,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-10T09:18:58,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731230338238, max timestamp=1731230338247Num uniques 1013; 2024-11-10T09:18:58,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-10T09:18:58,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731230338238, max timestamp=1731230338247Num uniques 1013; 2024-11-10T09:18:58,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-10T09:18:58,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731230338238, max timestamp=1731230338247Num uniques 1013; 2024-11-10T09:18:58,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-10T09:18:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:18:58,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=182, cellSize=8322, totalHeapSize=19168, min timestamp=1731230338273, max timestamp=1731230338291Num uniques 1022; 2024-11-10T09:18:58,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46792, new segment data size=8322, suffix heap size=134256,new segment heap size=19168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=182 2024-11-10T09:18:58,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10152, totalHeapSize=23328, min timestamp=1731230338273, max timestamp=1731230338291Num uniques 1022; 2024-11-10T09:18:58,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46792, new segment data size=10152, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-10T09:18:58,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10152, totalHeapSize=23328, min timestamp=1731230338273, max timestamp=1731230338291Num uniques 1022; 2024-11-10T09:18:58,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46792, new segment data size=10152, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-10T09:18:58,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731230338335, max timestamp=1731230338336Num uniques 1081; 2024-11-10T09:18:58,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731230338335, max timestamp=1731230338336Num uniques 1041; 2024-11-10T09:18:58,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731230338335, max timestamp=1731230338336Num uniques 1081; 2024-11-10T09:18:58,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47676, new segment data size=8286, suffix heap size=136232,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1041, new segment cells count=181 2024-11-10T09:18:58,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-10T09:18:58,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-10T09:18:58,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=390, cellSize=17860, totalHeapSize=40800, min timestamp=1731230338352, max timestamp=1731230338362Num uniques 1040; 2024-11-10T09:18:58,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=17860, suffix heap size=136128,new segment heap size=40800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=390 2024-11-10T09:18:58,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=390, cellSize=17860, totalHeapSize=40800, min timestamp=1731230338352, max timestamp=1731230338362Num uniques 1040; 2024-11-10T09:18:58,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=17860, suffix heap size=136128,new segment heap size=40800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=390 2024-11-10T09:18:58,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=390, cellSize=17860, totalHeapSize=40800, min timestamp=1731230338352, max timestamp=1731230338362Num uniques 1040; 2024-11-10T09:18:58,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=17860, suffix heap size=136128,new segment heap size=40800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=390 2024-11-10T09:18:58,393 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/516035b0768f4458b9eaba445fded3ee as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/516035b0768f4458b9eaba445fded3ee 2024-11-10T09:18:58,411 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into 516035b0768f4458b9eaba445fded3ee(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:18:58,411 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:58,411 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230337556; duration=0sec 2024-11-10T09:18:58,411 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:18:58,411 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:18:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230398419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230398421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230398422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230398423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230398423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,444 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d8cb461a4f70416aa3964fc8e756bdd8 2024-11-10T09:18:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/571af45a65eb4473bfd95c296d1acb32 is 50, key is test_row_0/C:col10/1731230336605/Put/seqid=0 2024-11-10T09:18:58,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742472_1648 (size=12301) 2024-11-10T09:18:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230398538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230398538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230398538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230398538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230398538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230398747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230398747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230398747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230398747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230398747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:58,853 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/571af45a65eb4473bfd95c296d1acb32 2024-11-10T09:18:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/96e1ec9ee0484ad9bc2884ca13169d15 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15 2024-11-10T09:18:58,861 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15, entries=150, sequenceid=401, filesize=30.5 K 2024-11-10T09:18:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/d8cb461a4f70416aa3964fc8e756bdd8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d8cb461a4f70416aa3964fc8e756bdd8 2024-11-10T09:18:58,873 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d8cb461a4f70416aa3964fc8e756bdd8, entries=150, sequenceid=401, filesize=12.0 K 2024-11-10T09:18:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/571af45a65eb4473bfd95c296d1acb32 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/571af45a65eb4473bfd95c296d1acb32 2024-11-10T09:18:58,877 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/571af45a65eb4473bfd95c296d1acb32, entries=150, sequenceid=401, filesize=12.0 K 2024-11-10T09:18:58,878 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for 38741dfb7340c00e6e9f2028384a14b8 in 1280ms, sequenceid=401, compaction requested=false 2024-11-10T09:18:58,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:18:58,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:58,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T09:18:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T09:18:58,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T09:18:58,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-10T09:18:58,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7430 sec 2024-11-10T09:18:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731230338419, max timestamp=1731230339060Num uniques 1249; 2024-11-10T09:18:59,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=167.72 KB heapSize=463.64 KB 2024-11-10T09:18:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=57204, new segment data size=9114, suffix heap size=157864,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1249, new segment cells count=199 2024-11-10T09:18:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731230338419, max timestamp=1731230339060Num uniques 1249; 2024-11-10T09:18:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731230338419, max timestamp=1731230339060Num uniques 1249; 2024-11-10T09:18:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57204, new segment data size=9114, suffix heap size=157864,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1249, new segment cells count=199 2024-11-10T09:18:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=57204, new segment data size=9114, suffix heap size=157864,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1249, new segment cells count=199 2024-11-10T09:18:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:18:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:18:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:18:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:18:59,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c5d460dd32e48fab067c00e6b705c29_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230339062/Put/seqid=0 2024-11-10T09:18:59,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742473_1649 (size=12454) 2024-11-10T09:18:59,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230339082, max timestamp=1731230339092Num uniques 859; 2024-11-10T09:18:59,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230339082, max timestamp=1731230339092Num uniques 859; 2024-11-10T09:18:59,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731230339082, max timestamp=1731230339092Num uniques 859; 2024-11-10T09:18:59,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:18:59,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:18:59,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T09:18:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230399112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230399113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230399113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230399120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230399120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230399227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230399227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230399227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230399237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230399237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:18:59,287 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:18:59,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:18:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T09:18:59,289 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:18:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:18:59,289 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:18:59,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:18:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:18:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230399437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230399437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230399437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:18:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230399447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230399447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:18:59,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c5d460dd32e48fab067c00e6b705c29_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c5d460dd32e48fab067c00e6b705c29_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:18:59,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/67b268b616c140f6bc05a5ac9e1dd2e9, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:18:59,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/67b268b616c140f6bc05a5ac9e1dd2e9 is 175, key is test_row_0/A:col10/1731230339062/Put/seqid=0 2024-11-10T09:18:59,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742474_1650 (size=31255) 2024-11-10T09:18:59,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:18:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:18:59,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:18:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230399747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230399747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230399748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230399757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:18:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230399757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:18:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:18:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:18:59,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=599, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/67b268b616c140f6bc05a5ac9e1dd2e9 2024-11-10T09:18:59,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/0e5ee47cee7c4aafae57ce781454bda0 is 50, key is test_row_0/B:col10/1731230339062/Put/seqid=0 2024-11-10T09:18:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742475_1651 (size=12301) 2024-11-10T09:18:59,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:18:59,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:18:59,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:18:59,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:18:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:19:00,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:19:00,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:00,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:00,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:00,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:19:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230400257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230400257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230400268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230400268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230400268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:00,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/0e5ee47cee7c4aafae57ce781454bda0 2024-11-10T09:19:00,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/4012426aa29e4da6b919e3a1b4968a43 is 50, key is test_row_0/C:col10/1731230339062/Put/seqid=0 2024-11-10T09:19:00,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742476_1652 (size=12301) 2024-11-10T09:19:00,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/4012426aa29e4da6b919e3a1b4968a43 2024-11-10T09:19:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/67b268b616c140f6bc05a5ac9e1dd2e9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9 2024-11-10T09:19:00,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9, entries=150, sequenceid=599, filesize=30.5 K 2024-11-10T09:19:00,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/0e5ee47cee7c4aafae57ce781454bda0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/0e5ee47cee7c4aafae57ce781454bda0 2024-11-10T09:19:00,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/0e5ee47cee7c4aafae57ce781454bda0, entries=150, sequenceid=599, filesize=12.0 K 2024-11-10T09:19:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/4012426aa29e4da6b919e3a1b4968a43 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/4012426aa29e4da6b919e3a1b4968a43 2024-11-10T09:19:00,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/4012426aa29e4da6b919e3a1b4968a43, entries=150, sequenceid=599, filesize=12.0 K 2024-11-10T09:19:00,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~122.20 KB/125136, currentSize=147.60 KB/151140 for 38741dfb7340c00e6e9f2028384a14b8 in 1255ms, sequenceid=599, compaction requested=true 2024-11-10T09:19:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:19:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:00,319 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:19:00,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:00,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:00,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:19:00,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:19:00,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:00,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:00,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:19:00,320 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:19:00,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,320 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,320 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5525b6434b3d433093aac88692d404b0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d8cb461a4f70416aa3964fc8e756bdd8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/0e5ee47cee7c4aafae57ce781454bda0] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.6 K 2024-11-10T09:19:00,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/516035b0768f4458b9eaba445fded3ee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=92.2 K 2024-11-10T09:19:00,320 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,320 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/516035b0768f4458b9eaba445fded3ee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9] 2024-11-10T09:19:00,321 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 5525b6434b3d433093aac88692d404b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230335980 2024-11-10T09:19:00,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516035b0768f4458b9eaba445fded3ee, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230335980 2024-11-10T09:19:00,321 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting d8cb461a4f70416aa3964fc8e756bdd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731230336600 2024-11-10T09:19:00,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e1ec9ee0484ad9bc2884ca13169d15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731230336600 2024-11-10T09:19:00,321 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5ee47cee7c4aafae57ce781454bda0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731230339061 2024-11-10T09:19:00,321 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67b268b616c140f6bc05a5ac9e1dd2e9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731230339061 2024-11-10T09:19:00,327 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:00,328 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#548 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:00,328 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111008bde7d9624341dfbca9dbc01a38ba19_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:00,328 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/14397adf1cf74fe7abfac06e5cb14fe9 is 50, key is test_row_0/B:col10/1731230339062/Put/seqid=0 2024-11-10T09:19:00,329 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111008bde7d9624341dfbca9dbc01a38ba19_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:00,330 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111008bde7d9624341dfbca9dbc01a38ba19_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:00,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742478_1654 (size=13017) 2024-11-10T09:19:00,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742477_1653 (size=4469) 2024-11-10T09:19:00,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,358 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:19:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:19:00,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110022cc298f38b4a9a8c23faadd2f0479a_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230339111/Put/seqid=0 2024-11-10T09:19:00,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742479_1655 (size=12454) 2024-11-10T09:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:19:00,741 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/14397adf1cf74fe7abfac06e5cb14fe9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/14397adf1cf74fe7abfac06e5cb14fe9 2024-11-10T09:19:00,749 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into 14397adf1cf74fe7abfac06e5cb14fe9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:00,749 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:00,749 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230340319; duration=0sec 2024-11-10T09:19:00,749 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:19:00,749 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:19:00,749 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:00,749 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#549 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:00,750 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/8caa9a2070cd40579ca94cc893605425 is 175, key is test_row_0/A:col10/1731230339062/Put/seqid=0 2024-11-10T09:19:00,751 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:00,751 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:19:00,751 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:00,751 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/437b54178f4e4d2eba39ffe62765f67c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/571af45a65eb4473bfd95c296d1acb32, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/4012426aa29e4da6b919e3a1b4968a43] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.6 K 2024-11-10T09:19:00,751 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 437b54178f4e4d2eba39ffe62765f67c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230335980 2024-11-10T09:19:00,751 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 571af45a65eb4473bfd95c296d1acb32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731230336600 2024-11-10T09:19:00,753 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 4012426aa29e4da6b919e3a1b4968a43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731230339061 2024-11-10T09:19:00,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:00,771 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110022cc298f38b4a9a8c23faadd2f0479a_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110022cc298f38b4a9a8c23faadd2f0479a_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3c1dc4b73d8b416cae582a6eac6813dc, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3c1dc4b73d8b416cae582a6eac6813dc is 175, key is test_row_0/A:col10/1731230339111/Put/seqid=0 2024-11-10T09:19:00,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742480_1656 (size=31971) 2024-11-10T09:19:00,782 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#551 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:00,783 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/59a69aa6424848b684d58d48ebff99e8 is 50, key is test_row_0/C:col10/1731230339062/Put/seqid=0 2024-11-10T09:19:00,786 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/8caa9a2070cd40579ca94cc893605425 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8caa9a2070cd40579ca94cc893605425 2024-11-10T09:19:00,792 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into 8caa9a2070cd40579ca94cc893605425(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:00,793 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:00,793 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230340319; duration=0sec 2024-11-10T09:19:00,793 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:00,793 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:19:00,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742481_1657 (size=31255) 2024-11-10T09:19:00,797 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=636, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3c1dc4b73d8b416cae582a6eac6813dc 2024-11-10T09:19:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/f8984abfbc96464caf68c23aea61d336 is 50, key is test_row_0/B:col10/1731230339111/Put/seqid=0 2024-11-10T09:19:00,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742482_1658 (size=12983) 2024-11-10T09:19:00,834 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/59a69aa6424848b684d58d48ebff99e8 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/59a69aa6424848b684d58d48ebff99e8 2024-11-10T09:19:00,838 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into 59a69aa6424848b684d58d48ebff99e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:00,838 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:00,838 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=13, startTime=1731230340320; duration=0sec 2024-11-10T09:19:00,838 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:00,838 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:19:00,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742483_1659 (size=12301) 2024-11-10T09:19:01,244 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/f8984abfbc96464caf68c23aea61d336 2024-11-10T09:19:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/3229450b296d448dbf4c12783cbd67ed is 50, key is test_row_0/C:col10/1731230339111/Put/seqid=0 2024-11-10T09:19:01,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742484_1660 (size=12301) 2024-11-10T09:19:01,256 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/3229450b296d448dbf4c12783cbd67ed 2024-11-10T09:19:01,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/3c1dc4b73d8b416cae582a6eac6813dc as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc 2024-11-10T09:19:01,266 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc, entries=150, sequenceid=636, filesize=30.5 K 2024-11-10T09:19:01,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/f8984abfbc96464caf68c23aea61d336 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/f8984abfbc96464caf68c23aea61d336 2024-11-10T09:19:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,270 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/f8984abfbc96464caf68c23aea61d336, entries=150, sequenceid=636, filesize=12.0 K 2024-11-10T09:19:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/3229450b296d448dbf4c12783cbd67ed as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/3229450b296d448dbf4c12783cbd67ed 2024-11-10T09:19:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,273 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/3229450b296d448dbf4c12783cbd67ed, entries=150, sequenceid=636, filesize=12.0 K 2024-11-10T09:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=0 B/0 for 38741dfb7340c00e6e9f2028384a14b8 in 916ms, sequenceid=636, compaction requested=false 2024-11-10T09:19:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T09:19:01,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9870 sec 2024-11-10T09:19:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9910 sec 2024-11-10T09:19:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T09:19:01,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:19:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:01,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:19:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:01,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:19:01,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103fb6936b5fff41138157bd84aa4f5791_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742485_1661 (size=22618) 2024-11-10T09:19:01,335 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:01,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103fb6936b5fff41138157bd84aa4f5791_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103fb6936b5fff41138157bd84aa4f5791_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:01,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0ad67d2303194fdda3090b3f6ddff114, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:01,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0ad67d2303194fdda3090b3f6ddff114 is 175, key is test_row_0/A:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:01,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230341345, max timestamp=1731230341349Num uniques 859; 2024-11-10T09:19:01,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:19:01,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230341345, max timestamp=1731230341349Num uniques 859; 2024-11-10T09:19:01,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:19:01,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230341345, max timestamp=1731230341349Num uniques 859; 2024-11-10T09:19:01,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:19:01,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742486_1662 (size=66023) 2024-11-10T09:19:01,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230401370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230401371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731230341366, max timestamp=1731230341377Num uniques 1018; 2024-11-10T09:19:01,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T09:19:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731230341366, max timestamp=1731230341377Num uniques 1018; 2024-11-10T09:19:01,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T09:19:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731230341366, max timestamp=1731230341377Num uniques 1018; 2024-11-10T09:19:01,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-10T09:19:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230401440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:19:01,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:19:01,448 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:19:01,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230401442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-10T09:19:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230401443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:01,450 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:19:01,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:19:01,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:19:01,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230401507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230401507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230401557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230401558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230401558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230401717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230401717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=649, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0ad67d2303194fdda3090b3f6ddff114 2024-11-10T09:19:01,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/cf0a659c3f994bef9001d122c0457e52 is 50, key is test_row_0/B:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:01,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742487_1663 (size=12301) 2024-11-10T09:19:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230401768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230401768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230401768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:01,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:01,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230402027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230402027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:02,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230402078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230402078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230402078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/cf0a659c3f994bef9001d122c0457e52 2024-11-10T09:19:02,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/059587239d7f4cc2a3355922fc015df4 is 50, key is test_row_0/C:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:02,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742488_1664 (size=12301) 2024-11-10T09:19:02,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:02,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:19:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230402537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230402538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/059587239d7f4cc2a3355922fc015df4 2024-11-10T09:19:02,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/0ad67d2303194fdda3090b3f6ddff114 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114 2024-11-10T09:19:02,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114, entries=350, sequenceid=649, filesize=64.5 K 2024-11-10T09:19:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/cf0a659c3f994bef9001d122c0457e52 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/cf0a659c3f994bef9001d122c0457e52 2024-11-10T09:19:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:02,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/cf0a659c3f994bef9001d122c0457e52, entries=150, sequenceid=649, filesize=12.0 K 2024-11-10T09:19:02,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/059587239d7f4cc2a3355922fc015df4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/059587239d7f4cc2a3355922fc015df4 2024-11-10T09:19:02,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230402588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230402589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230402589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:02,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/059587239d7f4cc2a3355922fc015df4, entries=150, sequenceid=649, filesize=12.0 K 2024-11-10T09:19:02,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.62 KB/138872 for 38741dfb7340c00e6e9f2028384a14b8 in 1285ms, sequenceid=649, compaction requested=true 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:02,599 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:02,599 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:19:02,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:19:02,605 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:02,605 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:19:02,606 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,606 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,606 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/14397adf1cf74fe7abfac06e5cb14fe9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/f8984abfbc96464caf68c23aea61d336, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/cf0a659c3f994bef9001d122c0457e52] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.7 K 2024-11-10T09:19:02,606 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8caa9a2070cd40579ca94cc893605425, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=126.2 K 2024-11-10T09:19:02,606 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8caa9a2070cd40579ca94cc893605425, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114] 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14397adf1cf74fe7abfac06e5cb14fe9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731230339061 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8caa9a2070cd40579ca94cc893605425, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731230339061 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8984abfbc96464caf68c23aea61d336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1731230339108 2024-11-10T09:19:02,606 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c1dc4b73d8b416cae582a6eac6813dc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1731230339108 2024-11-10T09:19:02,607 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ad67d2303194fdda3090b3f6ddff114, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731230341294 2024-11-10T09:19:02,607 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0a659c3f994bef9001d122c0457e52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731230341295 2024-11-10T09:19:02,611 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:02,612 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#557 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:02,612 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e9ca310bc6644cb786f5c03e17dc6a07 is 50, key is test_row_0/B:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:02,614 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105f829c7c0c1045c69036bc598fd889de_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:02,615 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105f829c7c0c1045c69036bc598fd889de_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:02,616 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105f829c7c0c1045c69036bc598fd889de_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:02,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742489_1665 (size=13119) 2024-11-10T09:19:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742490_1666 (size=4469) 2024-11-10T09:19:02,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T09:19:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:02,672 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=135.62 KB heapSize=382.70 KB 2024-11-10T09:19:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:19:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:19:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:19:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:19:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:19:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T09:19:02,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109eb4d6c7670b4922a5e6541644b9f43e_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230341438/Put/seqid=0 2024-11-10T09:19:02,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742491_1667 (size=12454) 2024-11-10T09:19:03,041 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#558 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:03,041 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e9ca310bc6644cb786f5c03e17dc6a07 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e9ca310bc6644cb786f5c03e17dc6a07 2024-11-10T09:19:03,041 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/edee5b24a2aa4e83b99de1811be09e63 is 175, key is test_row_0/A:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:03,046 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into e9ca310bc6644cb786f5c03e17dc6a07(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:03,046 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:03,046 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230342599; duration=0sec 2024-11-10T09:19:03,046 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:19:03,046 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:19:03,046 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:03,048 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:03,048 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:19:03,048 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:03,048 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/59a69aa6424848b684d58d48ebff99e8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/3229450b296d448dbf4c12783cbd67ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/059587239d7f4cc2a3355922fc015df4] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.7 K 2024-11-10T09:19:03,049 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a69aa6424848b684d58d48ebff99e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731230339061 2024-11-10T09:19:03,050 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3229450b296d448dbf4c12783cbd67ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1731230339108 2024-11-10T09:19:03,050 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059587239d7f4cc2a3355922fc015df4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731230341295 2024-11-10T09:19:03,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742492_1668 (size=32073) 2024-11-10T09:19:03,060 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#560 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:03,060 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/44f2b8605841460d8a8b1b0b2079f079 is 50, key is test_row_0/C:col10/1731230341313/Put/seqid=0 2024-11-10T09:19:03,062 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/edee5b24a2aa4e83b99de1811be09e63 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/edee5b24a2aa4e83b99de1811be09e63 2024-11-10T09:19:03,066 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into edee5b24a2aa4e83b99de1811be09e63(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:03,066 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:03,066 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230342599; duration=0sec 2024-11-10T09:19:03,066 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:03,066 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:19:03,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742493_1669 (size=13085) 2024-11-10T09:19:03,070 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/44f2b8605841460d8a8b1b0b2079f079 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/44f2b8605841460d8a8b1b0b2079f079 2024-11-10T09:19:03,074 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into 44f2b8605841460d8a8b1b0b2079f079(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:03,074 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:03,074 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=13, startTime=1731230342599; duration=0sec 2024-11-10T09:19:03,074 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:03,074 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:19:03,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:03,093 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109eb4d6c7670b4922a5e6541644b9f43e_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109eb4d6c7670b4922a5e6541644b9f43e_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/630cef6fb0d44de39d1115cb777ea501, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/630cef6fb0d44de39d1115cb777ea501 is 175, key is test_row_0/A:col10/1731230341438/Put/seqid=0 2024-11-10T09:19:03,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742494_1670 (size=31255) 2024-11-10T09:19:03,506 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=702, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/630cef6fb0d44de39d1115cb777ea501 2024-11-10T09:19:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/8e84d6e44c264f688296f517c283b015 is 50, key is test_row_0/B:col10/1731230341438/Put/seqid=0 2024-11-10T09:19:03,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742495_1671 (size=12301) 2024-11-10T09:19:03,541 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/8e84d6e44c264f688296f517c283b015 2024-11-10T09:19:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:03,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. as already flushing 2024-11-10T09:19:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/f7c85b6262a546209dd38c2409196f4d is 50, key is test_row_0/C:col10/1731230341438/Put/seqid=0 2024-11-10T09:19:03,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742496_1672 (size=12301) 2024-11-10T09:19:03,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230403561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230403564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:03,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39002 deadline: 1731230403607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1731230403607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,610 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:19:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38996 deadline: 1731230403607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,610 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_0' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,611 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,38513,1731230168538 for 'test_row_2' in 38741dfb7340c00e6e9f2028384a14b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230403667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230403677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230403877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230403887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:03,957 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/f7c85b6262a546209dd38c2409196f4d 2024-11-10T09:19:03,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/630cef6fb0d44de39d1115cb777ea501 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501 2024-11-10T09:19:03,964 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501, entries=150, sequenceid=702, filesize=30.5 K 2024-11-10T09:19:03,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/8e84d6e44c264f688296f517c283b015 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e84d6e44c264f688296f517c283b015 2024-11-10T09:19:03,967 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e84d6e44c264f688296f517c283b015, entries=150, sequenceid=702, filesize=12.0 K 2024-11-10T09:19:03,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/f7c85b6262a546209dd38c2409196f4d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/f7c85b6262a546209dd38c2409196f4d 2024-11-10T09:19:03,970 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/f7c85b6262a546209dd38c2409196f4d, entries=150, sequenceid=702, filesize=12.0 K 2024-11-10T09:19:03,971 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~135.62 KB/138872, heapSize ~382.66 KB/391840, currentSize=46.96 KB/48090 for 38741dfb7340c00e6e9f2028384a14b8 in 1299ms, sequenceid=702, compaction requested=false 2024-11-10T09:19:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-10T09:19:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-10T09:19:03,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T09:19:03,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5210 sec 2024-11-10T09:19:03,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5260 sec 2024-11-10T09:19:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:04,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T09:19:04,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:19:04,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:04,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:19:04,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:04,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:19:04,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:04,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e1cffbcbcd84d61b6df49b761d8f33d_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:04,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742497_1673 (size=14994) 2024-11-10T09:19:04,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230344229, max timestamp=1731230344240Num uniques 859; 2024-11-10T09:19:04,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230344229, max timestamp=1731230344240Num uniques 859; 2024-11-10T09:19:04,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:19:04,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:19:04,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731230344229, max timestamp=1731230344240Num uniques 859; 2024-11-10T09:19:04,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/802346a334f5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T09:19:04,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230404287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230404287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230404397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230404397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,518 INFO [Thread-2540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2540. 2024-11-10T09:19:04,518 DEBUG [Thread-2540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:04,518 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,518 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:04,520 INFO [Thread-2534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2534. 2024-11-10T09:19:04,520 DEBUG [Thread-2534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:04,520 DEBUG [Thread-2534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,520 INFO [Thread-2536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2536. 2024-11-10T09:19:04,520 DEBUG [Thread-2536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:04,520 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:04,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:04,520 DEBUG [Thread-2534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,520 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,523 INFO [Thread-2542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2542. 2024-11-10T09:19:04,523 DEBUG [Thread-2542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:04,523 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,523 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:04,525 INFO [Thread-2538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2538. 2024-11-10T09:19:04,525 DEBUG [Thread-2538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:04,525 DEBUG [Thread-2538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,525 DEBUG [Thread-2538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:04,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230404607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230404607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:04,630 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e1cffbcbcd84d61b6df49b761d8f33d_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e1cffbcbcd84d61b6df49b761d8f33d_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:04,631 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/88d974caec464cb2bb0c71bb0da533a9, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:04,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/88d974caec464cb2bb0c71bb0da533a9 is 175, key is test_row_0/A:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:04,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742498_1674 (size=39949) 2024-11-10T09:19:04,636 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=716, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/88d974caec464cb2bb0c71bb0da533a9 2024-11-10T09:19:04,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/6d5ad640f7b0478f80f34f212f7b98e0 is 50, key is test_row_0/B:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:04,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742499_1675 (size=12301) 2024-11-10T09:19:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230404918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230404918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:05,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=716 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/6d5ad640f7b0478f80f34f212f7b98e0 2024-11-10T09:19:05,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4 is 50, key is test_row_0/C:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:05,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742500_1676 (size=12301) 2024-11-10T09:19:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39018 deadline: 1731230405427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38992 deadline: 1731230405427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 2024-11-10T09:19:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:19:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., hostname=802346a334f5,38513,1731230168538, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:19:05,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=716 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4 2024-11-10T09:19:05,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/88d974caec464cb2bb0c71bb0da533a9 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9 2024-11-10T09:19:05,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9, entries=200, sequenceid=716, filesize=39.0 K 2024-11-10T09:19:05,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/6d5ad640f7b0478f80f34f212f7b98e0 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6d5ad640f7b0478f80f34f212f7b98e0 2024-11-10T09:19:05,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6d5ad640f7b0478f80f34f212f7b98e0, entries=150, sequenceid=716, filesize=12.0 K 2024-11-10T09:19:05,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4 2024-11-10T09:19:05,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4, entries=150, sequenceid=716, filesize=12.0 K 2024-11-10T09:19:05,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 38741dfb7340c00e6e9f2028384a14b8 in 1272ms, sequenceid=716, compaction requested=true 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:05,465 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38741dfb7340c00e6e9f2028384a14b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:19:05,465 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:19:05,465 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:05,465 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/A is initiating minor compaction (all files) 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/B is initiating minor compaction (all files) 2024-11-10T09:19:05,466 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/B in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:05,466 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/A in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:05,466 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e9ca310bc6644cb786f5c03e17dc6a07, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e84d6e44c264f688296f517c283b015, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6d5ad640f7b0478f80f34f212f7b98e0] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.8 K 2024-11-10T09:19:05,466 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/edee5b24a2aa4e83b99de1811be09e63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=100.9 K 2024-11-10T09:19:05,466 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. files: [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/edee5b24a2aa4e83b99de1811be09e63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9] 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting e9ca310bc6644cb786f5c03e17dc6a07, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731230341295 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting edee5b24a2aa4e83b99de1811be09e63, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731230341295 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e84d6e44c264f688296f517c283b015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1731230341438 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 630cef6fb0d44de39d1115cb777ea501, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1731230341438 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5ad640f7b0478f80f34f212f7b98e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=716, earliestPutTs=1731230343549 2024-11-10T09:19:05,466 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d974caec464cb2bb0c71bb0da533a9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=716, earliestPutTs=1731230343549 2024-11-10T09:19:05,472 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#B#compaction#566 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:05,472 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/8e2f8adbb382490eb137240f55093291 is 50, key is test_row_0/B:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:05,474 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:05,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742501_1677 (size=13221) 2024-11-10T09:19:05,480 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110da6a738ea03c4ca3a555584b0cd66831_38741dfb7340c00e6e9f2028384a14b8 store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:05,497 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110da6a738ea03c4ca3a555584b0cd66831_38741dfb7340c00e6e9f2028384a14b8, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:05,498 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110da6a738ea03c4ca3a555584b0cd66831_38741dfb7340c00e6e9f2028384a14b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:05,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742502_1678 (size=4469) 2024-11-10T09:19:05,501 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#A#compaction#567 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:05,502 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/15bb9c3658fe47fdb4bc74f29a817e7d is 175, key is test_row_0/A:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:05,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742503_1679 (size=32175) 2024-11-10T09:19:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:19:05,618 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:19:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] regionserver.HRegion(8855): Flush requested on 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:05,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-10T09:19:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:19:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:19:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:19:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:19:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:19:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T09:19:05,630 INFO [Thread-2523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2523. 2024-11-10T09:19:05,630 DEBUG [Thread-2523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:05,630 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:05,630 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:05,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:05,632 INFO [Thread-2527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2527. 2024-11-10T09:19:05,632 DEBUG [Thread-2527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:05,632 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:05,632 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:05,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:05,639 INFO [Thread-2529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2529. 2024-11-10T09:19:05,639 DEBUG [Thread-2529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:05,639 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:05,639 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:05,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:05,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb06c632133841dbadb8d8b19c8115d1_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230344286/Put/seqid=0 2024-11-10T09:19:05,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742504_1680 (size=12454) 2024-11-10T09:19:05,882 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/8e2f8adbb382490eb137240f55093291 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e2f8adbb382490eb137240f55093291 2024-11-10T09:19:05,885 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/B of 38741dfb7340c00e6e9f2028384a14b8 into 8e2f8adbb382490eb137240f55093291(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:05,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:05,885 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/B, priority=13, startTime=1731230345465; duration=0sec 2024-11-10T09:19:05,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:19:05,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:B 2024-11-10T09:19:05,885 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:19:05,886 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:19:05,886 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1541): 38741dfb7340c00e6e9f2028384a14b8/C is initiating minor compaction (all files) 2024-11-10T09:19:05,886 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38741dfb7340c00e6e9f2028384a14b8/C in TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:05,886 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/44f2b8605841460d8a8b1b0b2079f079, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/f7c85b6262a546209dd38c2409196f4d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4] into tmpdir=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp, totalSize=36.8 K 2024-11-10T09:19:05,886 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f2b8605841460d8a8b1b0b2079f079, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731230341295 2024-11-10T09:19:05,886 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting f7c85b6262a546209dd38c2409196f4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1731230341438 2024-11-10T09:19:05,887 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f71a1e7cbd9467a8ae7e27b40d7b2f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=716, earliestPutTs=1731230343549 2024-11-10T09:19:05,892 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38741dfb7340c00e6e9f2028384a14b8#C#compaction#569 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:19:05,892 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/dc42c2cd3d864224918bc3e0679a198b is 50, key is test_row_0/C:col10/1731230343559/Put/seqid=0 2024-11-10T09:19:05,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742505_1681 (size=13187) 2024-11-10T09:19:05,909 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/15bb9c3658fe47fdb4bc74f29a817e7d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/15bb9c3658fe47fdb4bc74f29a817e7d 2024-11-10T09:19:05,912 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/A of 38741dfb7340c00e6e9f2028384a14b8 into 15bb9c3658fe47fdb4bc74f29a817e7d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:05,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:05,912 INFO [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/A, priority=13, startTime=1731230345465; duration=0sec 2024-11-10T09:19:05,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:05,912 DEBUG [RS:0;802346a334f5:38513-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:A 2024-11-10T09:19:06,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:06,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb06c632133841dbadb8d8b19c8115d1_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb06c632133841dbadb8d8b19c8115d1_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:06,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2f5abfbe1e384f8a863769cbd5445e37, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:06,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2f5abfbe1e384f8a863769cbd5445e37 is 175, key is test_row_0/A:col10/1731230344286/Put/seqid=0 2024-11-10T09:19:06,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742506_1682 (size=31255) 2024-11-10T09:19:06,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=755, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2f5abfbe1e384f8a863769cbd5445e37 2024-11-10T09:19:06,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/ba1ff3fc91344b008a033d2d86334b9b is 50, key is test_row_0/B:col10/1731230344286/Put/seqid=0 2024-11-10T09:19:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742507_1683 (size=12301) 2024-11-10T09:19:06,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/ba1ff3fc91344b008a033d2d86334b9b 2024-11-10T09:19:06,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/c9fa557b1dd24c1db7ba194a0da3540d is 50, key is test_row_0/C:col10/1731230344286/Put/seqid=0 2024-11-10T09:19:06,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742508_1684 (size=12301) 2024-11-10T09:19:06,298 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/dc42c2cd3d864224918bc3e0679a198b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dc42c2cd3d864224918bc3e0679a198b 2024-11-10T09:19:06,302 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38741dfb7340c00e6e9f2028384a14b8/C of 38741dfb7340c00e6e9f2028384a14b8 into dc42c2cd3d864224918bc3e0679a198b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:19:06,302 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:06,303 INFO [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., storeName=38741dfb7340c00e6e9f2028384a14b8/C, priority=13, startTime=1731230345465; duration=0sec 2024-11-10T09:19:06,303 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:19:06,303 DEBUG [RS:0;802346a334f5:38513-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38741dfb7340c00e6e9f2028384a14b8:C 2024-11-10T09:19:06,439 INFO [Thread-2531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2531. 2024-11-10T09:19:06,439 INFO [Thread-2525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2525. 2024-11-10T09:19:06,439 DEBUG [Thread-2531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:06,439 DEBUG [Thread-2525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:19:06,439 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:06,439 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:06,439 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:06,439 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:06,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:06,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 200 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2596 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2689 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2584 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2588 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2693 2024-11-10T09:19:06,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:19:06,440 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:19:06,440 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:19:06,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:06,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:06,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:06,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:19:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T09:19:06,443 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230346443"}]},"ts":"1731230346443"} 2024-11-10T09:19:06,444 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:19:06,444 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:19:06,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:19:06,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, UNASSIGN}] 2024-11-10T09:19:06,446 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, UNASSIGN 2024-11-10T09:19:06,447 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=CLOSING, regionLocation=802346a334f5,38513,1731230168538 2024-11-10T09:19:06,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, UNASSIGN because future has completed 2024-11-10T09:19:06,448 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:19:06,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538}] 2024-11-10T09:19:06,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/c9fa557b1dd24c1db7ba194a0da3540d 2024-11-10T09:19:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/2f5abfbe1e384f8a863769cbd5445e37 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2f5abfbe1e384f8a863769cbd5445e37 2024-11-10T09:19:06,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2f5abfbe1e384f8a863769cbd5445e37, entries=150, sequenceid=755, filesize=30.5 K 2024-11-10T09:19:06,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/ba1ff3fc91344b008a033d2d86334b9b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/ba1ff3fc91344b008a033d2d86334b9b 2024-11-10T09:19:06,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/ba1ff3fc91344b008a033d2d86334b9b, entries=150, sequenceid=755, filesize=12.0 K 2024-11-10T09:19:06,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/c9fa557b1dd24c1db7ba194a0da3540d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/c9fa557b1dd24c1db7ba194a0da3540d 2024-11-10T09:19:06,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/c9fa557b1dd24c1db7ba194a0da3540d, entries=150, sequenceid=755, filesize=12.0 K 2024-11-10T09:19:06,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=20.13 KB/20610 for 38741dfb7340c00e6e9f2028384a14b8 in 865ms, sequenceid=755, compaction requested=false 2024-11-10T09:19:06,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38741dfb7340c00e6e9f2028384a14b8: 2024-11-10T09:19:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T09:19:06,600 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 38741dfb7340c00e6e9f2028384a14b8, disabling compactions & flushes 2024-11-10T09:19:06,600 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. after waiting 0 ms 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:06,600 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 38741dfb7340c00e6e9f2028384a14b8 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=A 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=B 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38741dfb7340c00e6e9f2028384a14b8, store=C 2024-11-10T09:19:06,600 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T09:19:06,604 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c57994baa79d4e16b5ec7bea3584c1aa_38741dfb7340c00e6e9f2028384a14b8 is 50, key is test_row_0/A:col10/1731230345638/Put/seqid=0 2024-11-10T09:19:06,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742509_1685 (size=9914) 2024-11-10T09:19:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T09:19:06,806 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:19:07,007 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:19:07,010 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c57994baa79d4e16b5ec7bea3584c1aa_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c57994baa79d4e16b5ec7bea3584c1aa_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:07,011 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c057aaca8ff544cab21ce50a4f1b5401, store: [table=TestAcidGuarantees family=A region=38741dfb7340c00e6e9f2028384a14b8] 2024-11-10T09:19:07,012 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c057aaca8ff544cab21ce50a4f1b5401 is 175, key is test_row_0/A:col10/1731230345638/Put/seqid=0 2024-11-10T09:19:07,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742510_1686 (size=22561) 2024-11-10T09:19:07,016 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=764, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c057aaca8ff544cab21ce50a4f1b5401 2024-11-10T09:19:07,021 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e407e579b9cb460cb9eb117df8e8fd18 is 50, key is test_row_0/B:col10/1731230345638/Put/seqid=0 2024-11-10T09:19:07,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742511_1687 (size=9857) 2024-11-10T09:19:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T09:19:07,424 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=764 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e407e579b9cb460cb9eb117df8e8fd18 2024-11-10T09:19:07,428 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/704d6daed75642afa4e5d07a8b4c45e4 is 50, key is test_row_0/C:col10/1731230345638/Put/seqid=0 2024-11-10T09:19:07,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742512_1688 (size=9857) 2024-11-10T09:19:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T09:19:07,836 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=764 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/704d6daed75642afa4e5d07a8b4c45e4 2024-11-10T09:19:07,840 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/A/c057aaca8ff544cab21ce50a4f1b5401 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c057aaca8ff544cab21ce50a4f1b5401 2024-11-10T09:19:07,842 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c057aaca8ff544cab21ce50a4f1b5401, entries=100, sequenceid=764, filesize=22.0 K 2024-11-10T09:19:07,843 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/B/e407e579b9cb460cb9eb117df8e8fd18 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e407e579b9cb460cb9eb117df8e8fd18 2024-11-10T09:19:07,846 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e407e579b9cb460cb9eb117df8e8fd18, entries=100, sequenceid=764, filesize=9.6 K 2024-11-10T09:19:07,846 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/.tmp/C/704d6daed75642afa4e5d07a8b4c45e4 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/704d6daed75642afa4e5d07a8b4c45e4 2024-11-10T09:19:07,849 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/704d6daed75642afa4e5d07a8b4c45e4, entries=100, sequenceid=764, filesize=9.6 K 2024-11-10T09:19:07,849 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 38741dfb7340c00e6e9f2028384a14b8 in 1249ms, sequenceid=764, compaction requested=true 2024-11-10T09:19:07,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8fc19a67b6a94a5c97ca6d90f8a3a920, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/19a6727c33ab41a3873e47092a1d0133, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/7182a94c639a4da9baa9830e2083007c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/bc7c6bd62a964b02a3ee53367912bd20, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c34eda6003804439901e27bc64152634, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/516035b0768f4458b9eaba445fded3ee, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8caa9a2070cd40579ca94cc893605425, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/edee5b24a2aa4e83b99de1811be09e63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9] to archive 2024-11-10T09:19:07,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:19:07,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/adaa3d95c6d7498a974f964b4de21230 2024-11-10T09:19:07,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1f6e94249ba84d46a0dfe041b691f71d 2024-11-10T09:19:07,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8fc19a67b6a94a5c97ca6d90f8a3a920 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8fc19a67b6a94a5c97ca6d90f8a3a920 2024-11-10T09:19:07,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2cc7a84ebbd043b898ec2f3513981d45 2024-11-10T09:19:07,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0a1f5f9f73d745148e09a1a5e18f656d 2024-11-10T09:19:07,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/e956b1665f464c30b9930d6db10da5f2 2024-11-10T09:19:07,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/19a6727c33ab41a3873e47092a1d0133 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/19a6727c33ab41a3873e47092a1d0133 2024-11-10T09:19:07,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/65f9ad56d55748f4b48954e12b2795c0 2024-11-10T09:19:07,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/405316df1eb1414294861d13fc2c5a90 2024-11-10T09:19:07,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/7182a94c639a4da9baa9830e2083007c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/7182a94c639a4da9baa9830e2083007c 2024-11-10T09:19:07,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/1db9429e8430431eb0c6552269288103 2024-11-10T09:19:07,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/336627bebb50427da23d9386ea59c373 2024-11-10T09:19:07,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/bc7c6bd62a964b02a3ee53367912bd20 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/bc7c6bd62a964b02a3ee53367912bd20 2024-11-10T09:19:07,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3e73ef7f6d464559a7e8a5a6bed7d83b 2024-11-10T09:19:07,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c51636c4c62640f19a19b2740a1ba4b6 2024-11-10T09:19:07,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c34eda6003804439901e27bc64152634 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c34eda6003804439901e27bc64152634 2024-11-10T09:19:07,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/6272cfdf2d5d414a8a89516d56af6285 2024-11-10T09:19:07,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/516035b0768f4458b9eaba445fded3ee to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/516035b0768f4458b9eaba445fded3ee 2024-11-10T09:19:07,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/ef34087d78324afc8530cb7156e37cf6 2024-11-10T09:19:07,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/96e1ec9ee0484ad9bc2884ca13169d15 2024-11-10T09:19:07,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8caa9a2070cd40579ca94cc893605425 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/8caa9a2070cd40579ca94cc893605425 2024-11-10T09:19:07,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/67b268b616c140f6bc05a5ac9e1dd2e9 2024-11-10T09:19:07,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/3c1dc4b73d8b416cae582a6eac6813dc 2024-11-10T09:19:07,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/0ad67d2303194fdda3090b3f6ddff114 2024-11-10T09:19:07,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/edee5b24a2aa4e83b99de1811be09e63 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/edee5b24a2aa4e83b99de1811be09e63 2024-11-10T09:19:07,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/630cef6fb0d44de39d1115cb777ea501 2024-11-10T09:19:07,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/88d974caec464cb2bb0c71bb0da533a9 2024-11-10T09:19:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/c72849c73bed472da2158f5d7244d24c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/7b4da0e73c9a4f15b19d45e4500ed2ba, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d27876400f0d4a43add757778d6fa197, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/83e6df9961464553b0c6bd8d4d5c2fb0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/085bc5ca9a084498aee8211074d9f8b8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d00f74f59c514fae9362000753e10083, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5bdd5c3c321746f8a2186c2f9117a108, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/76b8d9bc2a784fdbb0f3352329b4be95, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/627529f55343452ca516fc15b00e615c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/df43d80d3bd748e89e0846ff900c5687, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/03977c60f9cd49c08901a2777f91fa63, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/b4039eac32b741bc95910b3cc8bb8b38, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/66d68a9aff884886954e5eb6c224d488, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/2cd8a8b2e6184a1baa1902b146efdcf1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e798bb4dbc6e425eaef15192a576c644, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6a997519e93e41cd8a32e0f8b33805de, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/100743fbc25142c68c57a6d2e6d4a8d7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5525b6434b3d433093aac88692d404b0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d0afdf12ed6e40508842f6c47a8058c1, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d8cb461a4f70416aa3964fc8e756bdd8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/14397adf1cf74fe7abfac06e5cb14fe9, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/0e5ee47cee7c4aafae57ce781454bda0, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/f8984abfbc96464caf68c23aea61d336, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e9ca310bc6644cb786f5c03e17dc6a07, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/cf0a659c3f994bef9001d122c0457e52, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e84d6e44c264f688296f517c283b015, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6d5ad640f7b0478f80f34f212f7b98e0] to archive 2024-11-10T09:19:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:19:07,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/c72849c73bed472da2158f5d7244d24c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/c72849c73bed472da2158f5d7244d24c 2024-11-10T09:19:07,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/7b4da0e73c9a4f15b19d45e4500ed2ba to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/7b4da0e73c9a4f15b19d45e4500ed2ba 2024-11-10T09:19:07,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d27876400f0d4a43add757778d6fa197 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d27876400f0d4a43add757778d6fa197 2024-11-10T09:19:07,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/83e6df9961464553b0c6bd8d4d5c2fb0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/83e6df9961464553b0c6bd8d4d5c2fb0 2024-11-10T09:19:07,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/085bc5ca9a084498aee8211074d9f8b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/085bc5ca9a084498aee8211074d9f8b8 2024-11-10T09:19:07,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d00f74f59c514fae9362000753e10083 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d00f74f59c514fae9362000753e10083 2024-11-10T09:19:07,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5bdd5c3c321746f8a2186c2f9117a108 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5bdd5c3c321746f8a2186c2f9117a108 2024-11-10T09:19:07,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/76b8d9bc2a784fdbb0f3352329b4be95 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/76b8d9bc2a784fdbb0f3352329b4be95 2024-11-10T09:19:07,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/627529f55343452ca516fc15b00e615c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/627529f55343452ca516fc15b00e615c 2024-11-10T09:19:07,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/df43d80d3bd748e89e0846ff900c5687 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/df43d80d3bd748e89e0846ff900c5687 2024-11-10T09:19:07,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/03977c60f9cd49c08901a2777f91fa63 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/03977c60f9cd49c08901a2777f91fa63 2024-11-10T09:19:07,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/b4039eac32b741bc95910b3cc8bb8b38 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/b4039eac32b741bc95910b3cc8bb8b38 2024-11-10T09:19:07,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/66d68a9aff884886954e5eb6c224d488 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/66d68a9aff884886954e5eb6c224d488 2024-11-10T09:19:07,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/2cd8a8b2e6184a1baa1902b146efdcf1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/2cd8a8b2e6184a1baa1902b146efdcf1 2024-11-10T09:19:07,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e798bb4dbc6e425eaef15192a576c644 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e798bb4dbc6e425eaef15192a576c644 2024-11-10T09:19:07,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6a997519e93e41cd8a32e0f8b33805de to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6a997519e93e41cd8a32e0f8b33805de 2024-11-10T09:19:07,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/100743fbc25142c68c57a6d2e6d4a8d7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/100743fbc25142c68c57a6d2e6d4a8d7 2024-11-10T09:19:07,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5525b6434b3d433093aac88692d404b0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/5525b6434b3d433093aac88692d404b0 2024-11-10T09:19:07,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d0afdf12ed6e40508842f6c47a8058c1 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d0afdf12ed6e40508842f6c47a8058c1 2024-11-10T09:19:07,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d8cb461a4f70416aa3964fc8e756bdd8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/d8cb461a4f70416aa3964fc8e756bdd8 2024-11-10T09:19:07,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/14397adf1cf74fe7abfac06e5cb14fe9 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/14397adf1cf74fe7abfac06e5cb14fe9 2024-11-10T09:19:07,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/0e5ee47cee7c4aafae57ce781454bda0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/0e5ee47cee7c4aafae57ce781454bda0 2024-11-10T09:19:07,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/f8984abfbc96464caf68c23aea61d336 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/f8984abfbc96464caf68c23aea61d336 2024-11-10T09:19:07,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e9ca310bc6644cb786f5c03e17dc6a07 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e9ca310bc6644cb786f5c03e17dc6a07 2024-11-10T09:19:07,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/cf0a659c3f994bef9001d122c0457e52 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/cf0a659c3f994bef9001d122c0457e52 2024-11-10T09:19:07,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e84d6e44c264f688296f517c283b015 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e84d6e44c264f688296f517c283b015 2024-11-10T09:19:07,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6d5ad640f7b0478f80f34f212f7b98e0 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/6d5ad640f7b0478f80f34f212f7b98e0 2024-11-10T09:19:07,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/1923b46a014f4db79cdefcd92a8b110e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7534425065a848f3a43a61f201b9d940, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/0fe8e1830999474b902dc6c77fa89956, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/083719572c0b4107aae50826ab150351, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/16b73c10ec7b4b6491308d03677fa0f6, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/28007b6bbce64e33b07137da620fa3bf, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/58882a6a88a6491d984c1d0ef56df622, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/794bb24332de4894a7a975eb792fe994, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/e868c09327154d419a046b26bb94bc06, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7c5cbd13e3e048a18250f4e4fa3797e2, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/42534e5711874dcba08eb50a486856cb, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/81c62e573b924ae3ac589248a737a875, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/75b50b44ef5944b181d42cff984de012, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/eb365a65e53e47498e686f4ebe54171d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dca880b578f148abb952666c725d413e, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/a1cdcae567de497b8b37d992241aace7, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/437b54178f4e4d2eba39ffe62765f67c, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/73b45f0f9cb14353b4f2c739269e4d77, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/571af45a65eb4473bfd95c296d1acb32, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/59a69aa6424848b684d58d48ebff99e8, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/4012426aa29e4da6b919e3a1b4968a43, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/3229450b296d448dbf4c12783cbd67ed, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/44f2b8605841460d8a8b1b0b2079f079, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/059587239d7f4cc2a3355922fc015df4, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/f7c85b6262a546209dd38c2409196f4d, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4] to archive 2024-11-10T09:19:07,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:19:07,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/1923b46a014f4db79cdefcd92a8b110e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/1923b46a014f4db79cdefcd92a8b110e 2024-11-10T09:19:07,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7534425065a848f3a43a61f201b9d940 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7534425065a848f3a43a61f201b9d940 2024-11-10T09:19:07,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/0fe8e1830999474b902dc6c77fa89956 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/0fe8e1830999474b902dc6c77fa89956 2024-11-10T09:19:07,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/083719572c0b4107aae50826ab150351 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/083719572c0b4107aae50826ab150351 2024-11-10T09:19:07,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/16b73c10ec7b4b6491308d03677fa0f6 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/16b73c10ec7b4b6491308d03677fa0f6 2024-11-10T09:19:07,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/28007b6bbce64e33b07137da620fa3bf to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/28007b6bbce64e33b07137da620fa3bf 2024-11-10T09:19:07,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/58882a6a88a6491d984c1d0ef56df622 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/58882a6a88a6491d984c1d0ef56df622 2024-11-10T09:19:07,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/794bb24332de4894a7a975eb792fe994 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/794bb24332de4894a7a975eb792fe994 2024-11-10T09:19:07,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/e868c09327154d419a046b26bb94bc06 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/e868c09327154d419a046b26bb94bc06 2024-11-10T09:19:07,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7c5cbd13e3e048a18250f4e4fa3797e2 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/7c5cbd13e3e048a18250f4e4fa3797e2 2024-11-10T09:19:07,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/42534e5711874dcba08eb50a486856cb to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/42534e5711874dcba08eb50a486856cb 2024-11-10T09:19:07,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/81c62e573b924ae3ac589248a737a875 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/81c62e573b924ae3ac589248a737a875 2024-11-10T09:19:07,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/75b50b44ef5944b181d42cff984de012 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/75b50b44ef5944b181d42cff984de012 2024-11-10T09:19:07,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/eb365a65e53e47498e686f4ebe54171d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/eb365a65e53e47498e686f4ebe54171d 2024-11-10T09:19:07,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dca880b578f148abb952666c725d413e to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dca880b578f148abb952666c725d413e 2024-11-10T09:19:07,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/a1cdcae567de497b8b37d992241aace7 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/a1cdcae567de497b8b37d992241aace7 2024-11-10T09:19:07,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/437b54178f4e4d2eba39ffe62765f67c to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/437b54178f4e4d2eba39ffe62765f67c 2024-11-10T09:19:07,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/73b45f0f9cb14353b4f2c739269e4d77 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/73b45f0f9cb14353b4f2c739269e4d77 2024-11-10T09:19:07,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/571af45a65eb4473bfd95c296d1acb32 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/571af45a65eb4473bfd95c296d1acb32 2024-11-10T09:19:07,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/59a69aa6424848b684d58d48ebff99e8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/59a69aa6424848b684d58d48ebff99e8 2024-11-10T09:19:07,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/4012426aa29e4da6b919e3a1b4968a43 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/4012426aa29e4da6b919e3a1b4968a43 2024-11-10T09:19:07,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/3229450b296d448dbf4c12783cbd67ed to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/3229450b296d448dbf4c12783cbd67ed 2024-11-10T09:19:07,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/44f2b8605841460d8a8b1b0b2079f079 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/44f2b8605841460d8a8b1b0b2079f079 2024-11-10T09:19:07,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/059587239d7f4cc2a3355922fc015df4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/059587239d7f4cc2a3355922fc015df4 2024-11-10T09:19:07,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/f7c85b6262a546209dd38c2409196f4d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/f7c85b6262a546209dd38c2409196f4d 2024-11-10T09:19:07,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/6f71a1e7cbd9467a8ae7e27b40d7b2f4 2024-11-10T09:19:07,913 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/recovered.edits/767.seqid, newMaxSeqId=767, maxSeqId=4 2024-11-10T09:19:07,913 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. 2024-11-10T09:19:07,913 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 38741dfb7340c00e6e9f2028384a14b8: Waiting for close lock at 1731230346600Running coprocessor pre-close hooks at 1731230346600Disabling compacts and flushes for region at 1731230346600Disabling writes for close at 1731230346600Obtaining lock to block concurrent updates at 1731230346600Preparing flush snapshotting stores in 38741dfb7340c00e6e9f2028384a14b8 at 1731230346600Finished memstore snapshotting TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731230346600Flushing stores of TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8. at 1731230346601 (+1 ms)Flushing 38741dfb7340c00e6e9f2028384a14b8/A: creating writer at 1731230346601Flushing mob file 38741dfb7340c00e6e9f2028384a14b8/A: appending metadata at 1731230346604 (+3 ms)Flushing mob file 38741dfb7340c00e6e9f2028384a14b8/A: closing flushed file at 1731230346604Flushing 38741dfb7340c00e6e9f2028384a14b8/A: appending metadata at 1731230347011 (+407 ms)Flushing 38741dfb7340c00e6e9f2028384a14b8/A: closing flushed file at 1731230347011Flushing 38741dfb7340c00e6e9f2028384a14b8/B: creating writer at 1731230347019 (+8 ms)Flushing 38741dfb7340c00e6e9f2028384a14b8/B: appending metadata at 1731230347020 (+1 ms)Flushing 38741dfb7340c00e6e9f2028384a14b8/B: closing flushed file at 1731230347020Flushing 38741dfb7340c00e6e9f2028384a14b8/C: creating writer at 1731230347427 (+407 ms)Flushing 38741dfb7340c00e6e9f2028384a14b8/C: appending metadata at 1731230347428 (+1 ms)Flushing 38741dfb7340c00e6e9f2028384a14b8/C: closing flushed file at 1731230347428Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f66da8e: reopening flushed file at 1731230347839 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@571df8f1: reopening flushed file at 1731230347843 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a352cb4: reopening flushed file at 1731230347846 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 38741dfb7340c00e6e9f2028384a14b8 in 1249ms, sequenceid=764, compaction requested=true at 1731230347849 (+3 ms)Writing region close event to WAL at 1731230347911 (+62 ms)Running coprocessor post-close hooks at 1731230347913 (+2 ms)Closed at 1731230347913 2024-11-10T09:19:07,915 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:07,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=38741dfb7340c00e6e9f2028384a14b8, regionState=CLOSED 2024-11-10T09:19:07,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 because future has completed 2024-11-10T09:19:07,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-10T09:19:07,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 38741dfb7340c00e6e9f2028384a14b8, server=802346a334f5,38513,1731230168538 in 1.4680 sec 2024-11-10T09:19:07,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-10T09:19:07,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38741dfb7340c00e6e9f2028384a14b8, UNASSIGN in 1.4710 sec 2024-11-10T09:19:07,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T09:19:07,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-10T09:19:07,921 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230347921"}]},"ts":"1731230347921"} 2024-11-10T09:19:07,921 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:19:07,921 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:19:07,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4810 sec 2024-11-10T09:19:08,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:19:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T09:19:08,587 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:19:08,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:19:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:08,589 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T09:19:08,590 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:08,592 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,594 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C, FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/recovered.edits] 2024-11-10T09:19:08,596 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/15bb9c3658fe47fdb4bc74f29a817e7d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/15bb9c3658fe47fdb4bc74f29a817e7d 2024-11-10T09:19:08,596 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2f5abfbe1e384f8a863769cbd5445e37 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/2f5abfbe1e384f8a863769cbd5445e37 2024-11-10T09:19:08,597 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c057aaca8ff544cab21ce50a4f1b5401 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/A/c057aaca8ff544cab21ce50a4f1b5401 2024-11-10T09:19:08,598 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e2f8adbb382490eb137240f55093291 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/8e2f8adbb382490eb137240f55093291 2024-11-10T09:19:08,599 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/ba1ff3fc91344b008a033d2d86334b9b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/ba1ff3fc91344b008a033d2d86334b9b 2024-11-10T09:19:08,600 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e407e579b9cb460cb9eb117df8e8fd18 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/B/e407e579b9cb460cb9eb117df8e8fd18 2024-11-10T09:19:08,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/704d6daed75642afa4e5d07a8b4c45e4 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/704d6daed75642afa4e5d07a8b4c45e4 2024-11-10T09:19:08,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/c9fa557b1dd24c1db7ba194a0da3540d to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/c9fa557b1dd24c1db7ba194a0da3540d 2024-11-10T09:19:08,602 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dc42c2cd3d864224918bc3e0679a198b to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/C/dc42c2cd3d864224918bc3e0679a198b 2024-11-10T09:19:08,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/recovered.edits/767.seqid to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8/recovered.edits/767.seqid 2024-11-10T09:19:08,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/default/TestAcidGuarantees/38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,604 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:19:08,604 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:19:08,605 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T09:19:08,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110022cc298f38b4a9a8c23faadd2f0479a_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110022cc298f38b4a9a8c23faadd2f0479a_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101da8c8e1a74e4f219b3f35f3a539631f_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101da8c8e1a74e4f219b3f35f3a539631f_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,608 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e1cffbcbcd84d61b6df49b761d8f33d_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e1cffbcbcd84d61b6df49b761d8f33d_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,609 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111031238bfd41294b2196f778d75cbdb8b6_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111031238bfd41294b2196f778d75cbdb8b6_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,609 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032d52cd90bb14fe0918932f73b3b1731_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032d52cd90bb14fe0918932f73b3b1731_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c5d460dd32e48fab067c00e6b705c29_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c5d460dd32e48fab067c00e6b705c29_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,611 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103fb6936b5fff41138157bd84aa4f5791_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103fb6936b5fff41138157bd84aa4f5791_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,611 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110664b1a58e16e4859948e9c362318fb6f_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110664b1a58e16e4859948e9c362318fb6f_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107397e1dbdc2245b5a561ee8acc465b8a_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107397e1dbdc2245b5a561ee8acc465b8a_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108912d62c5de24e13b6129218b8ac40b9_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108912d62c5de24e13b6129218b8ac40b9_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,613 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109eb4d6c7670b4922a5e6541644b9f43e_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109eb4d6c7670b4922a5e6541644b9f43e_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a85ab10828ba41d18ac51b4b4dfd4354_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a85ab10828ba41d18ac51b4b4dfd4354_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b66e4c3d65604127a59a848706459c31_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b66e4c3d65604127a59a848706459c31_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,615 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b6e704baba9f4536b249f7479a06ac12_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b6e704baba9f4536b249f7479a06ac12_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,615 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8d5ac34e7634c479dd90394a2f63778_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8d5ac34e7634c479dd90394a2f63778_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,616 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c16bee5e1c3143c6ba75d059886eab1f_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c16bee5e1c3143c6ba75d059886eab1f_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,617 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c57994baa79d4e16b5ec7bea3584c1aa_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c57994baa79d4e16b5ec7bea3584c1aa_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,618 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7434a6c7aae4d2e85d77bb0f13a53bb_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7434a6c7aae4d2e85d77bb0f13a53bb_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ea9e50e2da7e4286a0174ccd44d7ae35_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ea9e50e2da7e4286a0174ccd44d7ae35_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb06c632133841dbadb8d8b19c8115d1_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb06c632133841dbadb8d8b19c8115d1_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,620 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f64066b6c5d94cdb8aa856d1b9c66d47_38741dfb7340c00e6e9f2028384a14b8 to hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f64066b6c5d94cdb8aa856d1b9c66d47_38741dfb7340c00e6e9f2028384a14b8 2024-11-10T09:19:08,620 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:19:08,622 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:08,623 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:19:08,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:19:08,626 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:08,626 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:19:08,626 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230348626"}]},"ts":"9223372036854775807"} 2024-11-10T09:19:08,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:19:08,627 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 38741dfb7340c00e6e9f2028384a14b8, NAME => 'TestAcidGuarantees,,1731230321315.38741dfb7340c00e6e9f2028384a14b8.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:19:08,627 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:19:08,627 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230348627"}]},"ts":"9223372036854775807"} 2024-11-10T09:19:08,629 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:19:08,629 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:19:08,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-10T09:19:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T09:19:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:19:08,699 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:19:08,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=455 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=684 (was 747), ProcessCount=11 (was 11), AvailableMemoryMB=1083 (was 1514) 2024-11-10T09:19:08,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T09:19:08,705 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:19:08,705 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:19:08,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:08,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:08,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:19:08,705 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T09:19:08,705 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-10T09:19:08,706 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=802346a334f5,43719,1731230167759 2024-11-10T09:19:08,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T09:19:08,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T09:19:08,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:19:08,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:19:08,771 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T09:19:08,771 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:19:08,771 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:19:08,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:08,772 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '802346a334f5,38513,1731230168538' ***** 2024-11-10T09:19:08,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:19:08,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:19:08,772 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T09:19:08,772 INFO [RS:0;802346a334f5:38513 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T09:19:08,772 INFO [RS:0;802346a334f5:38513 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T09:19:08,772 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T09:19:08,772 INFO [RS:0;802346a334f5:38513 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T09:19:08,772 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(959): stopping server 802346a334f5,38513,1731230168538 2024-11-10T09:19:08,772 INFO [RS:0;802346a334f5:38513 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T09:19:08,772 INFO [RS:0;802346a334f5:38513 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;802346a334f5:38513. 2024-11-10T09:19:08,772 DEBUG [RS:0;802346a334f5:38513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:19:08,772 DEBUG [RS:0;802346a334f5:38513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:08,773 INFO [RS:0;802346a334f5:38513 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T09:19:08,773 INFO [RS:0;802346a334f5:38513 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T09:19:08,773 INFO [RS:0;802346a334f5:38513 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T09:19:08,773 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T09:19:08,773 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T09:19:08,773 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T09:19:08,773 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:08,773 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T09:19:08,774 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T09:19:08,774 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T09:19:08,774 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T09:19:08,774 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T09:19:08,774 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T09:19:08,798 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/info/bf11a918c15243af9916af120a6c2d25 is 95, key is TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d./info:/1731230200426/DeleteFamily/seqid=0 2024-11-10T09:19:08,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742513_1689 (size=6514) 2024-11-10T09:19:08,870 INFO [regionserver/802346a334f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T09:19:08,974 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:09,174 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:09,202 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/info/bf11a918c15243af9916af120a6c2d25 2024-11-10T09:19:09,218 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/ns/beffa0e130db44e7aecd62d4fb389876 is 93, key is TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d./ns:/1731230200380/DeleteFamily/seqid=0 2024-11-10T09:19:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742514_1690 (size=5906) 2024-11-10T09:19:09,374 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:09,524 INFO [regionserver/802346a334f5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T09:19:09,524 INFO [regionserver/802346a334f5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T09:19:09,574 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:09,621 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/ns/beffa0e130db44e7aecd62d4fb389876 2024-11-10T09:19:09,643 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/rep_barrier/ada5dd4e82a443cbbcd05b1e1c5e360b is 102, key is TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d./rep_barrier:/1731230200380/DeleteFamily/seqid=0 2024-11-10T09:19:09,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742515_1691 (size=6025) 2024-11-10T09:19:09,774 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:09,975 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:10,046 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/rep_barrier/ada5dd4e82a443cbbcd05b1e1c5e360b 2024-11-10T09:19:10,062 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/table/dd48f6fd8baa4909bc6b0dc54b8f105d is 96, key is TestAcidGuarantees,,1731230173356.bfb4913e15c1cd99308737d49021ea5d./table:/1731230200380/DeleteFamily/seqid=0 2024-11-10T09:19:10,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742516_1692 (size=5936) 2024-11-10T09:19:10,175 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:10,375 DEBUG [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:19:10,465 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/table/dd48f6fd8baa4909bc6b0dc54b8f105d 2024-11-10T09:19:10,468 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/info/bf11a918c15243af9916af120a6c2d25 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/info/bf11a918c15243af9916af120a6c2d25 2024-11-10T09:19:10,472 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/info/bf11a918c15243af9916af120a6c2d25, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T09:19:10,473 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/ns/beffa0e130db44e7aecd62d4fb389876 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/ns/beffa0e130db44e7aecd62d4fb389876 2024-11-10T09:19:10,476 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/ns/beffa0e130db44e7aecd62d4fb389876, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T09:19:10,476 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/rep_barrier/ada5dd4e82a443cbbcd05b1e1c5e360b as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/rep_barrier/ada5dd4e82a443cbbcd05b1e1c5e360b 2024-11-10T09:19:10,479 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/rep_barrier/ada5dd4e82a443cbbcd05b1e1c5e360b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T09:19:10,480 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/.tmp/table/dd48f6fd8baa4909bc6b0dc54b8f105d as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/table/dd48f6fd8baa4909bc6b0dc54b8f105d 2024-11-10T09:19:10,482 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/table/dd48f6fd8baa4909bc6b0dc54b8f105d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T09:19:10,483 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-11-10T09:19:10,486 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T09:19:10,487 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T09:19:10,487 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T09:19:10,487 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731230348773Running coprocessor pre-close hooks at 1731230348773Disabling compacts and flushes for region at 1731230348773Disabling writes for close at 1731230348774 (+1 ms)Obtaining lock to block concurrent updates at 1731230348774Preparing flush snapshotting stores in 1588230740 at 1731230348774Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731230348775 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731230348775Flushing 1588230740/info: creating writer at 1731230348775Flushing 1588230740/info: appending metadata at 1731230348798 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731230348798Flushing 1588230740/ns: creating writer at 1731230349205 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731230349217 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731230349217Flushing 1588230740/rep_barrier: creating writer at 1731230349624 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731230349642 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731230349642Flushing 1588230740/table: creating writer at 1731230350049 (+407 ms)Flushing 1588230740/table: appending metadata at 1731230350061 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731230350061Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b349e5c: reopening flushed file at 1731230350468 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ee94df9: reopening flushed file at 1731230350472 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5296f3fe: reopening flushed file at 1731230350476 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71659d9f: reopening flushed file at 1731230350479 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1731230350483 (+4 ms)Writing region close event to WAL at 1731230350485 (+2 ms)Running coprocessor post-close hooks at 1731230350487 (+2 ms)Closed at 1731230350487 2024-11-10T09:19:10,487 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T09:19:10,575 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(976): stopping server 802346a334f5,38513,1731230168538; all regions closed. 2024-11-10T09:19:10,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741834_1010 (size=25344) 2024-11-10T09:19:10,581 DEBUG [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/oldWALs 2024-11-10T09:19:10,581 INFO [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 802346a334f5%2C38513%2C1731230168538.meta:.meta(num 1731230172769) 2024-11-10T09:19:10,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741832_1008 (size=21389688) 2024-11-10T09:19:10,585 DEBUG [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/oldWALs 2024-11-10T09:19:10,585 INFO [RS:0;802346a334f5:38513 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 802346a334f5%2C38513%2C1731230168538:(num 1731230171683) 2024-11-10T09:19:10,585 DEBUG [RS:0;802346a334f5:38513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:19:10,585 INFO [RS:0;802346a334f5:38513 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T09:19:10,585 INFO [RS:0;802346a334f5:38513 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T09:19:10,585 INFO [RS:0;802346a334f5:38513 {}] hbase.ChoreService(370): Chore service for: regionserver/802346a334f5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T09:19:10,585 INFO [RS:0;802346a334f5:38513 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T09:19:10,585 INFO [regionserver/802346a334f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T09:19:10,586 INFO [RS:0;802346a334f5:38513 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38513 2024-11-10T09:19:10,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/802346a334f5,38513,1731230168538 2024-11-10T09:19:10,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T09:19:10,658 INFO [RS:0;802346a334f5:38513 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T09:19:10,659 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [802346a334f5,38513,1731230168538] 2024-11-10T09:19:10,790 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/802346a334f5,38513,1731230168538 already deleted, retry=false 2024-11-10T09:19:10,791 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 802346a334f5,38513,1731230168538 expired; onlineServers=0 2024-11-10T09:19:10,791 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '802346a334f5,43719,1731230167759' ***** 2024-11-10T09:19:10,791 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T09:19:10,791 INFO [M:0;802346a334f5:43719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T09:19:10,791 INFO [M:0;802346a334f5:43719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T09:19:10,791 DEBUG [M:0;802346a334f5:43719 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T09:19:10,791 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T09:19:10,791 DEBUG [M:0;802346a334f5:43719 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T09:19:10,791 DEBUG [master/802346a334f5:0:becomeActiveMaster-HFileCleaner.large.0-1731230171382 {}] cleaner.HFileCleaner(306): Exit Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.large.0-1731230171382,5,FailOnTimeoutGroup] 2024-11-10T09:19:10,791 DEBUG [master/802346a334f5:0:becomeActiveMaster-HFileCleaner.small.0-1731230171385 {}] cleaner.HFileCleaner(306): Exit Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.small.0-1731230171385,5,FailOnTimeoutGroup] 2024-11-10T09:19:10,791 INFO [M:0;802346a334f5:43719 {}] hbase.ChoreService(370): Chore service for: master/802346a334f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T09:19:10,792 INFO [M:0;802346a334f5:43719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T09:19:10,792 DEBUG [M:0;802346a334f5:43719 {}] master.HMaster(1795): Stopping service threads 2024-11-10T09:19:10,792 INFO [M:0;802346a334f5:43719 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T09:19:10,792 INFO [M:0;802346a334f5:43719 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T09:19:10,792 ERROR [M:0;802346a334f5:43719 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T09:19:10,792 INFO [M:0;802346a334f5:43719 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T09:19:10,792 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T09:19:10,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T09:19:10,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:19:10,802 DEBUG [M:0;802346a334f5:43719 {}] zookeeper.ZKUtil(347): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T09:19:10,802 WARN [M:0;802346a334f5:43719 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T09:19:10,802 INFO [M:0;802346a334f5:43719 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/.lastflushedseqids 2024-11-10T09:19:10,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742517_1693 (size=72) 2024-11-10T09:19:10,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:19:10,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x1012407eb210001, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:19:10,891 INFO [RS:0;802346a334f5:38513 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T09:19:10,892 INFO [RS:0;802346a334f5:38513 {}] regionserver.HRegionServer(1031): Exiting; stopping=802346a334f5,38513,1731230168538; zookeeper connection closed. 2024-11-10T09:19:10,892 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@42e7760f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@42e7760f 2024-11-10T09:19:10,892 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T09:19:11,210 INFO [M:0;802346a334f5:43719 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T09:19:11,211 INFO [M:0;802346a334f5:43719 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T09:19:11,211 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T09:19:11,211 INFO [M:0;802346a334f5:43719 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:19:11,211 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:19:11,211 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T09:19:11,211 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:19:11,211 INFO [M:0;802346a334f5:43719 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=800.43 KB heapSize=982.11 KB 2024-11-10T09:19:11,224 DEBUG [M:0;802346a334f5:43719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e38395961b2481693cdde2f6302ebaa is 82, key is hbase:meta,,1/info:regioninfo/1731230172857/Put/seqid=0 2024-11-10T09:19:11,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742518_1694 (size=5672) 2024-11-10T09:19:11,628 INFO [M:0;802346a334f5:43719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2250 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e38395961b2481693cdde2f6302ebaa 2024-11-10T09:19:11,645 DEBUG [M:0;802346a334f5:43719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f553d470e404342806d9d6305349407 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1731230324379/Put/seqid=0 2024-11-10T09:19:11,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742519_1695 (size=45432) 2024-11-10T09:19:12,049 INFO [M:0;802346a334f5:43719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=799.88 KB at sequenceid=2250 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f553d470e404342806d9d6305349407 2024-11-10T09:19:12,052 INFO [M:0;802346a334f5:43719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f553d470e404342806d9d6305349407 2024-11-10T09:19:12,065 DEBUG [M:0;802346a334f5:43719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9fd9c50464eb4b2480410bdba7d3ed0e is 69, key is 802346a334f5,38513,1731230168538/rs:state/1731230171441/Put/seqid=0 2024-11-10T09:19:12,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073742520_1696 (size=5156) 2024-11-10T09:19:12,468 INFO [M:0;802346a334f5:43719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2250 (bloomFilter=true), to=hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9fd9c50464eb4b2480410bdba7d3ed0e 2024-11-10T09:19:12,472 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e38395961b2481693cdde2f6302ebaa as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1e38395961b2481693cdde2f6302ebaa 2024-11-10T09:19:12,475 INFO [M:0;802346a334f5:43719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1e38395961b2481693cdde2f6302ebaa, entries=8, sequenceid=2250, filesize=5.5 K 2024-11-10T09:19:12,475 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f553d470e404342806d9d6305349407 as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f553d470e404342806d9d6305349407 2024-11-10T09:19:12,478 INFO [M:0;802346a334f5:43719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f553d470e404342806d9d6305349407 2024-11-10T09:19:12,478 INFO [M:0;802346a334f5:43719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f553d470e404342806d9d6305349407, entries=170, sequenceid=2250, filesize=44.4 K 2024-11-10T09:19:12,478 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9fd9c50464eb4b2480410bdba7d3ed0e as hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9fd9c50464eb4b2480410bdba7d3ed0e 2024-11-10T09:19:12,480 INFO [M:0;802346a334f5:43719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38659/user/jenkins/test-data/84cb29e3-049d-e3d6-747a-c97cc7044a11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9fd9c50464eb4b2480410bdba7d3ed0e, entries=1, sequenceid=2250, filesize=5.0 K 2024-11-10T09:19:12,481 INFO [M:0;802346a334f5:43719 {}] regionserver.HRegion(3140): Finished flush of dataSize ~800.43 KB/819641, heapSize ~981.81 KB/1005376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1270ms, sequenceid=2250, compaction requested=false 2024-11-10T09:19:12,482 INFO [M:0;802346a334f5:43719 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:19:12,482 DEBUG [M:0;802346a334f5:43719 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731230351211Disabling compacts and flushes for region at 1731230351211Disabling writes for close at 1731230351211Obtaining lock to block concurrent updates at 1731230351211Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731230351211Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=819641, getHeapSize=1005616, getOffHeapSize=0, getCellsCount=2445 at 1731230351211Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731230351212 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731230351212Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731230351224 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731230351224Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731230351631 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731230351645 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731230351645Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731230352052 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731230352065 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731230352065Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79357607: reopening flushed file at 1731230352471 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ba0fc9: reopening flushed file at 1731230352475 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61ea5e18: reopening flushed file at 1731230352478 (+3 ms)Finished flush of dataSize ~800.43 KB/819641, heapSize ~981.81 KB/1005376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1270ms, sequenceid=2250, compaction requested=false at 1731230352481 (+3 ms)Writing region close event to WAL at 1731230352482 (+1 ms)Closed at 1731230352482 2024-11-10T09:19:12,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39443 is added to blk_1073741830_1006 (size=967090) 2024-11-10T09:19:12,485 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T09:19:12,485 INFO [M:0;802346a334f5:43719 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T09:19:12,485 INFO [M:0;802346a334f5:43719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43719 2024-11-10T09:19:12,485 INFO [M:0;802346a334f5:43719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T09:19:12,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:19:12,597 INFO [M:0;802346a334f5:43719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T09:19:12,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43719-0x1012407eb210000, quorum=127.0.0.1:53423, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:19:12,601 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T09:19:12,603 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T09:19:12,603 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T09:19:12,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T09:19:12,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/hadoop.log.dir/,STOPPED} 2024-11-10T09:19:12,606 WARN [BP-1750528295-172.17.0.3-1731230163282 heartbeating to localhost/127.0.0.1:38659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T09:19:12,606 WARN [BP-1750528295-172.17.0.3-1731230163282 heartbeating to localhost/127.0.0.1:38659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1750528295-172.17.0.3-1731230163282 (Datanode Uuid 93b0e4a0-6ef8-482f-96c7-e988491df6b4) service to localhost/127.0.0.1:38659 2024-11-10T09:19:12,606 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T09:19:12,606 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T09:19:12,609 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/data/data1/current/BP-1750528295-172.17.0.3-1731230163282 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T09:19:12,609 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/cluster_3c3e6463-dd39-b9b9-9769-42c8268f4e8e/data/data2/current/BP-1750528295-172.17.0.3-1731230163282 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T09:19:12,609 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T09:19:12,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T09:19:12,617 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T09:19:12,618 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T09:19:12,618 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T09:19:12,618 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/badc352b-7295-8b68-7bc4-4a793a9633cd/hadoop.log.dir/,STOPPED} 2024-11-10T09:19:12,639 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T09:19:12,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down